diff --git a/lua/mapvote/client/modules/map_vote.lua b/lua/mapvote/client/modules/map_vote.lua
index ace9500..0f54179 100644
--- a/lua/mapvote/client/modules/map_vote.lua
+++ b/lua/mapvote/client/modules/map_vote.lua
@@ -13,9 +13,7 @@ end
 function MapVote.ChangeVote( ply, mapIndex )
     if not IsValid( ply ) then return end
     if not IsValid( MapVote.Panel ) then return end
-
-    local mapData = MapVote.Panel.voteArea:GetMapDataByIndex( mapIndex )
-    MapVote.Panel.voteArea:SetVote( ply, mapData.map )
+    MapVote.Panel.voteArea:SetVote( ply, mapIndex )
 end
 
 function MapVote.StartVote( maps, endTime )
@@ -78,19 +76,18 @@ function MapVote.StartVote( maps, endTime )
 
         countdownLabel:SetText( string.FormattedTime( timeLeft or 0, "%02i:%02i" ) )
     end )
-    local voteArea = vgui.Create( "MapVote_Vote", frame ) --[[@as VoteArea]]
+    local voteArea = vgui.Create( "MapVote_VoteArea", frame )
     local margin = 10
     voteArea:Dock( FILL )
     voteArea:DockMargin( margin, margin, margin, margin )
-    voteArea:SetMaps( maps )
     voteArea:InvalidateLayout( true )
     voteArea:InvalidateParent( true )
+    voteArea:SetMaps( maps )
 
-    frame:SetTall( voteArea:GetTotalRowHeight() + infoRow:GetTall() + margin * 2 + 34 )
+    frame:SetTall( voteArea:GetTotalRowHeight() + infoRow:GetTall() + margin * 3 + 20 )
     frame:SetWide( voteArea:GetTotalRowWidth() + margin * 2 + 10 )
-    voteArea:InvalidateParent( true )
+    voteArea:InvalidateLayout( true )
     frame:Center()
-    voteArea:UpdateRowPositions()
 
     local lastClicked = CurTime()
     ---@diagnostic disable-next-line: duplicate-set-field
diff --git a/lua/mapvote/client/vgui/map_icon.lua b/lua/mapvote/client/vgui/map_icon.lua
index 0cd8165..72bbea6 100644
--- a/lua/mapvote/client/vgui/map_icon.lua
+++ b/lua/mapvote/client/vgui/map_icon.lua
@@ -1,4 +1,4 @@
----@class MapIcon : Panel
+---@class MapVote_MapIcon : Panel
 local PANEL = {}
 
 function PANEL:Init()
@@ -30,7 +30,7 @@ function PANEL:SetMap( map )
     MapVote.ThumbDownloader:QueueDownload( map, function( filepath )
         MapVote.TaskManager.AddFunc( function()
             ---@diagnostic disable-next-line: missing-parameter
-            self.button:SetImage( filepath or noIcon )
+            self.button:SetImage( filepath )
         end )
     end )
 end
diff --git a/lua/mapvote/client/vgui/mapvote_panel.lua b/lua/mapvote/client/vgui/mapvote_panel.lua
index e68937c..1f5dbe7 100644
--- a/lua/mapvote/client/vgui/mapvote_panel.lua
+++ b/lua/mapvote/client/vgui/mapvote_panel.lua
@@ -1,126 +1,178 @@
-local avatarSize = 30
-
-local function maxIconSize( w, h, n, _ )
-    local px = math.ceil( math.sqrt( n * w / h ) )
-    local sx, sy
-    if math.floor( px * h / w ) * px < n then
-        sx = h / math.ceil( px * h / w )
-    else
-        sx = w / px
-    end
-
-    local py = math.ceil( math.sqrt( n * h / w ) )
-    if math.floor( py * w / h ) * py < n then
-        sy = w / math.ceil( py * w / h )
-    else
-        sy = h / py
-    end
-
-    if sx < sy then
-        return sy, sy
-    end
-    return sx, sx
-end
-
----@class VoteArea : Panel
+---@class MapVote_VoteArea : Panel
 local PANEL = {}
 
+---@alias VoteAreaMap {name: string, panel: Panel, voters: Panel[]}
+---@alias VoteAreaVoter {identifier: string|Player, mapIndex: number, panel: Panel}
+
 function PANEL:Init()
+    self.avatarSize = 50
+
+    ---@type VoteAreaMap[]
     self.maps = {}
-    self.mapIndexes = {}
+
+    ---@type (Panel|{mapContainer: Panel})[]
     self.rows = {}
-    self.votes = {}
 
-    self.voteTally = {}
-end
+    ---@type table<string, VoteAreaVoter>
+    self.votes = {}
 
-function PANEL:Paint()
+    self.avatarIconPadding = 1
 end
 
-function PANEL:GetMapData( map )
-    local index = self.mapIndexes[map]
+function PANEL:GetMapByIndex( index )
     return self.maps[index]
 end
 
-function PANEL:GetMapDataByIndex( index )
-    return self.maps[index]
-end
+function PANEL:PerformLayout()
+    for _, row in ipairs( self.rows ) do
+        row.mapContainer:CenterHorizontal()
+    end
 
-function PANEL:SetMaps( maps )
-    self.maps = {}
-    for i, map in pairs( maps ) do
-        self.mapIndexes[map] = i
-        table.insert( self.maps, {
-            map = map,
-            panel = nil,
-            voterCount = 0,
-            hiddenCount = 0,
-        } )
+    -- This is expensive, but must be done so avatar positions dont get misaligned when parent panel is being minimized and resized
+    for _, map in ipairs( self.maps ) do
+        for i, voter in ipairs( map.voters ) do
+            local newX, newY, willOverflow = self:CalculateDesiredAvatarIconPosition( map, i )
+            voter:SetPos( newX, newY )
+            voter:SetVisible( not willOverflow )
+        end
     end
-    self:InvalidateLayout( true )
-    self:InvalidateParent( true )
-    self:setup()
 end
 
-function PANEL:SetVote( ply, mapName )
-    local mapData = self:GetMapData( mapName )
-    if not mapData then return end
+---@return number
+function PANEL:GetTotalRowWidth()
+    if #self.rows == 0 then return 0 end
+    return self.rows[1].mapContainer:GetWide()
+end
 
-    local iconContainer
-    local oldMapData
+---@return number
+function PANEL:GetTotalRowHeight()
+    if #self.rows == 0 then return 0 end
+    return #self.rows * self.rows[1]:GetTall()
+end
 
-    if self.votes[ply] then
-        local oldMapName = self.votes[ply].mapName
-        if oldMapName == mapName then return end
-        oldMapData = self:GetMapData( oldMapName )
+---@param identifier any
+---@param mapIndex number
+function PANEL:SetVote( identifier, mapIndex )
+    local mapData = self.maps[mapIndex]
+    if not mapData then
+        error( "Invalid map index " .. mapIndex )
+        return
+    end
 
-        oldMapData.voterCount = oldMapData.voterCount - 1
+    local oldVote = self.votes[identifier]
+    local panel
+    if oldVote then
+        if oldVote.mapIndex == mapIndex then
+            return
+        end
+        local oldMapData = self.maps[oldVote.mapIndex]
+
+        -- Find index in votes for the old map, then reposition all icons after it
+        local indexToRemove = nil
+        for i, voter in ipairs( oldMapData.voters ) do
+            if voter == oldVote.panel then
+                indexToRemove = i
+                break
+            end
+        end
+        if indexToRemove then
+            table.remove( oldMapData.voters, indexToRemove )
+        end
+
+        for i = indexToRemove, #oldMapData.voters do
+            local voter = oldMapData.voters[i]
+            local newX, newY, willOverflow = self:CalculateDesiredAvatarIconPosition( oldMapData, i )
+            voter:SetVisible( true )
+            voter:MoveTo( newX, newY, 0.2, nil, nil, function( _, pnl )
+                pnl:SetVisible( not willOverflow )
+            end )
+        end
 
-        iconContainer = self.votes[ply].panel
+        panel = oldVote.panel
     else
-        iconContainer = self:CreateVoterPanel( ply )
+        panel = self:CreateVoterPanel( identifier )
     end
 
-    local x, y, show = self:calculateDesiredAvatarIconPosition( mapData )
-    iconContainer:SetVisible( true )
-    iconContainer:MoveTo( x, y, 0.2, nil, nil, function( _, pnl )
-        pnl:SetVisible( show )
-    end )
+    table.insert( mapData.voters, panel )
 
-    mapData.voterCount = mapData.voterCount + 1
-    self.votes[ply] = {
-        mapName = mapName,
-        ply = ply,
-        panel = iconContainer,
+    self.votes[identifier] = {
+        identifier = identifier,
+        mapIndex = mapIndex,
+        panel = panel,
     }
-    if not oldMapData then return end
-    self:ResetAvatarPositions( oldMapData )
-    -- TODO icon container to show number of hidden votes
-end
 
-function PANEL:ResetAvatarPositions( mapData )
-    mapData.voterCount = 0
-    for _, voteData in pairs( self.votes ) do
-        if voteData.mapName == mapData.map then
-            local x, y, show = self:calculateDesiredAvatarIconPosition( mapData )
-            voteData.panel:SetVisible( show )
-            voteData.panel:MoveTo( x, y, 0.2, nil, nil, function( _, _ )
-            end )
-            mapData.voterCount = mapData.voterCount + 1
+    local newX, newY, willOverflow = self:CalculateDesiredAvatarIconPosition( mapData )
+    panel:SetVisible( true )
+    panel:MoveTo( newX, newY, 0.2, nil, nil, function()
+        if willOverflow then
+            panel:SetVisible( not willOverflow )
         end
+    end )
+end
+
+---@param mapData VoteAreaMap
+---@param index number|nil
+---@return number, number, boolean
+function PANEL:CalculateDesiredAvatarIconPosition( mapData, index )
+    if not index then
+        index = #mapData.voters
     end
+    index = index - 1
+
+    local avatarIconPadding = self.avatarIconPadding
+    local avatarTotalSize = self.avatarSize + avatarIconPadding * 2
+
+    local mapIcon = mapData.panel
+    local maxColumnCount = math.floor( mapIcon:GetWide() / avatarTotalSize )
+    local maxRowCount = math.floor( (mapIcon:GetTall() - 20) / avatarTotalSize )
+
+    local column = index % maxColumnCount
+    local row = math.floor( index / maxColumnCount )
+
+    local x = column * avatarTotalSize + avatarIconPadding
+    local y = row * avatarTotalSize + avatarIconPadding
+
+    local rootPosX, rootPosY = self:GetPositionRelativeToSelf( mapIcon )
+
+    return rootPosX + x, rootPosY + y, row >= maxRowCount
+end
+
+---@param mapPanel Panel
+---@return number, number
+---@private
+function PANEL:GetPositionRelativeToSelf( mapPanel )
+    local screenX, screenY = mapPanel:LocalToScreen( 0, 0 )
+    local x, y = self:ScreenToLocal( screenX, screenY )
+    return x, y
 end
 
-function PANEL:CreateVoterPanel( ply )
+---@param identifier string|Player
+---@return Player
+---@private
+function PANEL:GetPlayerFromIdentifier( identifier )
+    if type( identifier ) == "string" then
+        return player.GetBySteamID64( identifier )
+    elseif type( identifier ) == "Player" then
+        return identifier
+    end
+    return identifier
+end
+
+---@param identifier string|Player
+---@return Panel
+---@private
+function PANEL:CreateVoterPanel( identifier )
+    local ply = self:GetPlayerFromIdentifier( identifier )
+
     local iconContainer = vgui.Create( "Panel", self )
     local icon = vgui.Create( "AvatarImage", iconContainer ) --[[@as AvatarImage]]
-    icon:SetSize( avatarSize, avatarSize )
+    icon:SetSize( self.avatarSize, self.avatarSize )
     icon:SetZPos( 1000 )
 
     iconContainer.ply = ply
-    icon:SetPlayer( ply, avatarSize )
+    icon:SetPlayer( ply, self.avatarSize )
 
-    iconContainer:SetSize( avatarSize + 2, avatarSize + 2 )
+    iconContainer:SetSize( self.avatarSize + 2, self.avatarSize + 2 )
     icon:SetPos( 2, 2 )
 
     iconContainer:SetMouseInputEnabled( false )
@@ -129,104 +181,133 @@ function PANEL:CreateVoterPanel( ply )
     return iconContainer
 end
 
-function PANEL:calculateDesiredAvatarIconPosition( mapData )
-    local avatarIconPadding = 1
-    local avatarTotalSize = avatarSize + avatarIconPadding * 2
-
-    local mapIcon = mapData.panel
-    local maxColumnCount = math.floor( mapIcon:GetWide() / avatarTotalSize )
-    local maxRowCount = math.floor( (mapIcon:GetTall() - 10) / avatarTotalSize )
+---@param w number
+---@param h number
+---@param n number
+---@return number, number
+---@private
+function PANEL:maxIconSize( w, h, n )
+    local px = math.ceil( math.sqrt( n * w / h ) )
+    local sx, sy
+    if math.floor( px * h / w ) * px < n then
+        sx = h / math.ceil( px * h / w )
+    else
+        sx = w / px
+    end
 
-    -- calulate position of mapIcon relative to main vote area panel
-    local rowX, rowY = mapIcon.row:GetPos()
-    local iconX, iconY = mapIcon:GetPos()
-    local x, y = rowX + iconX, rowY + iconY
+    local py = math.ceil( math.sqrt( n * h / w ) )
+    if math.floor( py * w / h ) * py < n then
+        sy = w / math.ceil( py * w / h )
+    else
+        sy = h / py
+    end
 
-    local nextRowNumber = math.floor( mapData.voterCount / maxColumnCount )
-    if nextRowNumber >= (maxRowCount - 1) and mapData.voterCount >= maxRowCount * maxColumnCount then
-        return x + avatarTotalSize * (maxColumnCount - 1), y + (maxRowCount - 1) * avatarTotalSize, false
+    if sx < sy then
+        return sy, sy
     end
-    return x + avatarTotalSize * (mapData.voterCount % maxColumnCount), y + nextRowNumber * avatarTotalSize, true
+    return sx, sx
 end
 
-function PANEL:setup()
-    for _, row in pairs( self.rows ) do
+---@param maps string[]
+function PANEL:SetMaps( maps )
+    self.maps = {}
+    for i, map in ipairs( maps ) do
+        self.maps[i] = {
+            name = map,
+            voters = {},
+        }
+    end
+
+    -- Remove all rows
+    for _, row in ipairs( self.rows ) do
         row:Remove()
     end
-    self.rows = {}
-    local count = #self.maps
-    local margin = 2
-    local iconWidth, iconHeight = maxIconSize( self:GetWide(), self:GetTall(), count, 1 )
-    iconWidth = iconWidth - margin * 2
-
-    local maxItemsPerRow = math.floor( self:GetWide() / iconWidth )
-
-    local requiredRows = math.ceil( count / maxItemsPerRow )
-    for rowNumber = 1, requiredRows do
-        local itemsLeft = count - (rowNumber - 1) * maxItemsPerRow
-        local itemsInRow = math.min( maxItemsPerRow, itemsLeft )
-        local rowWidth = (iconWidth + margin * 2) * itemsInRow
-
-        local row = vgui.Create( "Panel", self )
-        table.insert( self.rows, row )
-        row:SetSize( rowWidth, iconHeight )
-        local extraSpace = math.max( 0, self:GetWide() - rowWidth )
-
-        row:SetPos( extraSpace / 2, iconHeight * (rowNumber - 1) )
-        for i = 1, itemsInRow do
-            local index = (rowNumber - 1) * maxItemsPerRow + i
-            local map = self.maps[index]
-            local mapIcon = vgui.Create( "MapVote_MapIcon", row ) --[[@as MapIcon]]
+
+    -- Remove all voters
+    for _, voter in pairs( self.votes ) do
+        voter.panel:Remove()
+    end
+    self.votes = {}
+
+    local maxW, maxH = self:maxIconSize( self:GetWide(), self:GetTall(), #self.maps )
+    local rowCount = math.floor( self:GetTall() / maxH )
+    local columnCount = math.floor( self:GetWide() / maxW )
+    self:CalculateAvatarSize( maxW, maxH )
+
+    local mapIndex = 1
+    for i = 1, rowCount do
+        local row = self:CreateRow( maxH )
+        self.rows[i] = row
+
+        -- create maps in row
+        local rowWidth = 0
+        for _ = 1, columnCount do
+            local currentMapIndex = mapIndex
+            local mapName = self.maps[currentMapIndex].name
+
+            local icon = vgui.Create( "MapVote_MapIcon", row.mapContainer )
+            icon:SetSize( maxW, maxH )
+            icon:Dock( LEFT )
+            icon:SetMap( mapName )
+            icon:DockMargin( 2, 2, 2, 2 )
             ---@diagnostic disable-next-line: duplicate-set-field
-            mapIcon.DoClick = function()
-                self:OnMapClicked( index, map )
+            icon.DoClick = function()
+                self:OnMapClicked( currentMapIndex, mapName )
             end
-            mapIcon:SetMap( map.map )
-            mapIcon:SetSize( iconWidth, iconHeight )
-            mapIcon:Dock( LEFT )
-            mapIcon:DockMargin( margin, margin, margin, margin )
-            mapIcon.row = row
-            mapIcon.voterCount = 0
-            map.panel = mapIcon
+
+            self.maps[mapIndex].panel = icon
+            rowWidth = rowWidth + maxW + 4
+            mapIndex = mapIndex + 1
+            if mapIndex > #self.maps then
+                break
+            end
+        end
+
+        row:Dock( TOP )
+        row:InvalidateParent( true )
+        row.mapContainer:SetWide( rowWidth )
+        row.mapContainer:CenterHorizontal()
+        if mapIndex > #self.maps then
+            break
         end
     end
 end
 
-function PANEL:UpdateRowPositions()
-    local count = #self.maps
-    local margin = 2
-    local iconWidth, iconHeight = maxIconSize( self:GetWide(), self:GetTall(), count, 1 )
-    iconWidth = iconWidth - margin * 2
+function PANEL:CalculateAvatarSize( maxW, _maxH )
+    -- leave space for 0 joins mid map vote
+    local avatarIconPadding = self.avatarIconPadding
+    local plyCount = math.max( player.GetCount(), 2 )
 
-    local maxItemsPerRow = math.floor( self:GetWide() / iconWidth )
-    for i, row in ipairs( self.rows ) do
-        local rowNumber = i
-        local itemsLeft = count - (rowNumber - 1) * maxItemsPerRow
-        local itemsInRow = math.min( maxItemsPerRow, itemsLeft )
-        local rowWidth = (iconWidth + margin * 2) * itemsInRow
+    -- add an extra row for title area
+    local rowCount = math.ceil( math.sqrt( plyCount ) ) + 1
 
-        local extraSpace = math.max( 0, self:GetWide() - rowWidth )
-
-        row:SetPos( extraSpace / 2, iconHeight * (rowNumber - 1) )
-    end
+    local availableSpace = maxW - (avatarIconPadding * 2) * rowCount
+    local newAvatarSize = math.ceil( availableSpace / rowCount ) - avatarIconPadding * 2
+    self.avatarSize = newAvatarSize
 end
 
-function PANEL:OnMapClicked( _, _ )
-    -- implement
+---@diagnostic disable-next-line: unused-local
+function PANEL:OnMapClicked( _index, _map )
 end
 
-function PANEL:GetTotalRowWidth()
-    if #self.rows == 0 then return 0 end
-    return self.rows[1]:GetWide()
-end
+---@return Panel|{mapContainer: Panel}
+---@private
+function PANEL:CreateRow( iconHeight )
+    local row = vgui.Create( "Panel", self )
+    row:DockMargin( 0, 0, 0, 0 )
+    row:SetTall( iconHeight )
+    row:InvalidateParent( true )
 
-function PANEL:GetTotalRowHeight()
-    if #self.rows == 0 then return 0 end
-    return #self.rows * self.rows[1]:GetTall()
+    -- we use an inner container so we can center the maps
+    local mapContainer = vgui.Create( "Panel", row )
+    mapContainer:SetTall( iconHeight )
+    row.mapContainer = mapContainer
+
+    return row
 end
 
 function PANEL:Flash( id )
-    local data = self:GetMapDataByIndex( id )
+    local data = self:GetMapByIndex( id )
     local panel = data.panel
     panel:SetBGColor( MapVote.style.colorPurple )
 
@@ -259,4 +340,4 @@ function PANEL:Flash( id )
     end )
 end
 
-vgui.Register( "MapVote_Vote", PANEL, "Panel" )
+vgui.Register( "MapVote_VoteArea", PANEL, "Panel" )
diff --git a/lua/mapvote/server/modules/net.lua b/lua/mapvote/server/modules/net.lua
index 677bbc7..f82e723 100644
--- a/lua/mapvote/server/modules/net.lua
+++ b/lua/mapvote/server/modules/net.lua
@@ -40,7 +40,10 @@ MapVote.Net.receiveWithMiddleware( "MapVote_ChangeVote", function( _, ply )
     if not IsValid( ply ) then return end
 
     local mapID = net.ReadUInt( 32 )
-    if not MapVote.state.currentMaps[mapID] then return end
+    if not MapVote.state.currentMaps[mapID] then
+        print( "MapVote: Player " .. ply:Nick() .. " tried to vote for invalid map " .. mapID )
+        return
+    end
 
     MapVote.state.votes[ply:SteamID()] = mapID