Skip to content

Commit

Permalink
Optimized DB
Browse files Browse the repository at this point in the history
  • Loading branch information
@JAKAMI99
JAKAMI99 committed May 24, 2024
1 parent 79e7f49 commit 02fa869
Show file tree
Hide file tree
Showing 4 changed files with 177 additions and 189 deletions.
1 change: 1 addition & 0 deletions app/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

#Logging config
from . import logging_config
from . import dbsetup

log = logging.getLogger(__name__)

Expand Down
236 changes: 114 additions & 122 deletions app/tools/geolocate_wigle.py
View file
Original file line number Diff line number Diff line change
@@ -1,148 +1,140 @@
import sqlite3, sys, requests
import sqlite3
import sys
import requests

def get_db_connection():
conn = sqlite3.connect('app/data/pwnamap.db')
conn.row_factory = sqlite3.Row
return conn

def create_pwned_table(sqlite_file):
# Connect to SQLite database
conn = get_db_connection()
cursor = conn.cursor()

# Create the 'pwned' table if it doesn't exist
cursor.execute('''
CREATE TABLE IF NOT EXISTS pwned (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
network_id TEXT,
encryption TEXT,
latitude REAL,
longitude REAL,
password TEXT,
UNIQUE (network_id)
)
''')

# Commit changes and close the connection
conn.commit()
conn.close()

def create_pwned_table():
with get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS pwned (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
network_id TEXT,
encryption TEXT,
latitude REAL,
longitude REAL,
password TEXT,
UNIQUE (network_id)
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS wigle (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
network_id TEXT UNIQUE,
encryption TEXT,
time TEXT,
signal REAL,
accuracy REAL,
network_type TEXT,
latitude REAL,
longitude REAL
)
''')
conn.commit()

def populate_pwned_data(api_key):
new_networks = 0 # Counter for new networks with geolocation
no_geolocation_networks = 0 # Counter for networks with no geolocation
total_networks = 0 # Counter for total processed Networks

# Connect to SQLite database
conn = get_db_connection()
cursor = conn.cursor()

try:
# Query wpasec table
cursor.execute('SELECT ap_mac, password FROM wpasec')
wpasec_data = cursor.fetchall()

# Query pwned table to get existing network_ids
cursor.execute('SELECT network_id FROM pwned')
existing_network_ids = {row[0] for row in cursor.fetchall()}

# Query WiGLE API and populate pwned table
for row in wpasec_data:
ap_mac = row[0]
password = row[1]

# Convert ap_mac to the format used in the WiGLE API

# Check if the entry already exists in pwned table
if ap_mac not in existing_network_ids:
total_networks += 1 # Increment total processed Networks counter


# Call WiGLE API to retrieve network information
response = requests.get(
f"https://api.wigle.net/api/v2/network/search?netid={ap_mac}",
headers={"Authorization": f"Basic {api_key}"})
if response.status_code == 401:
print(f"The Wigle API {api_key} Key is not authorized. Validate it in the Settings")
sys.exit
if response.status_code == 200:
wigle_data = response.json()
if 'results' in wigle_data and len(wigle_data['results']) > 0:

result = wigle_data['results'][0]
name = result.get('ssid')
network_id = result.get('netid')
encryption = result.get('encryption')
latitude = result.get('trilat')
longitude = result.get('trilong')
network_type = "WIFI" #Static value since only WIFI is expected (For local db)
time = result.get('lasttime')

print(f"✅📌 Found geolocation for {name}({ap_mac}).")


try:
cursor.execute('''
INSERT INTO pwned (name, network_id, encryption, latitude, longitude, password)
VALUES (?, ?, ?, ?, ?, ?)
''', (name, network_id, encryption, latitude, longitude, password))
new_networks += 1
except sqlite3.Error as e:
print(f"Got error {e}")
print(f"[PWNED]Got error {e} when inserting entry for network_id: {network_id}")

try:
cursor.execute('''
INSERT INTO wigle (name, network_id, encryption, latitude, longitude, network_type, time)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (name, network_id, encryption, latitude, longitude, network_type, time))
new_networks += 1
except sqlite3.Error as e:
print(f"[WIGLE]Got error {e} when inserting entry for network_id: {network_id}")
print(f"Error ")

new_networks = 0
no_geolocation_networks = 0
total_networks = 0

with get_db_connection() as conn:
cursor = conn.cursor()
try:
cursor.execute('SELECT ap_mac, password FROM wpasec')
wpasec_data = cursor.fetchall()

cursor.execute('SELECT network_id FROM pwned')
existing_network_ids = {row[0] for row in cursor.fetchall()}

for row in wpasec_data:
ap_mac, password = row

if ap_mac not in existing_network_ids:
total_networks += 1

response = requests.get(
f"https://api.wigle.net/api/v2/network/search?netid={ap_mac}",
headers={"Authorization": f"Basic {api_key}"}
)

if response.status_code == 401:
print(f"The Wigle API {api_key} Key is not authorized. Validate it in the Settings")
sys.exit(1)
elif response.status_code == 429:
print("❌ Received status code 429: API Limit reached.")
sys.exit(1)
elif response.status_code == 200:
wigle_data = response.json()
if 'results' in wigle_data and len(wigle_data['results']) > 0:
result = wigle_data['results'][0]
name = result.get('ssid')
network_id = result.get('netid')
encryption = result.get('encryption')
latitude = result.get('trilat')
longitude = result.get('trilong')
network_type = "WIFI"
time = result.get('lasttime')

print(f"✅📌 Found geolocation for {name}({ap_mac}).")

try:
cursor.execute('''
INSERT INTO pwned (name, network_id, encryption, latitude, longitude, password)
VALUES (?, ?, ?, ?, ?, ?)
''', (name, network_id, encryption, latitude, longitude, password))
new_networks += 1
except sqlite3.Error as e:
print(f"[PWNED] Got error {e} when inserting entry for network_id: {network_id}")

try:
cursor.execute('''
INSERT INTO wigle (name, network_id, encryption, latitude, longitude, network_type, time)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (name, network_id, encryption, latitude, longitude, network_type, time))
new_networks += 1
except sqlite3.Error as e:
print(f"[WIGLE] Got error {e} when inserting entry for network_id: {network_id}")
else:
no_geolocation_networks += 1
print(f"❌ No geolocation for {ap_mac} found...")
else:
no_geolocation_networks += 1
print(f"❌ No geolocation for {ap_mac} found...")
else:
print(f"Error retrieving data for {ap_mac}. Status code: {response.status_code}")

except Exception as e:
print("An error occurred:", str(e))
finally:
# Commit changes and close the connection
conn.commit()
conn.close()
print(f"Error retrieving data for {ap_mac}. Status code: {response.status_code}")
except Exception as e:
print("An error occurred:", str(e))
finally:
conn.commit()

return new_networks, no_geolocation_networks, total_networks

def get_key(username):
conn = get_db_connection()
cursor = conn.cursor()

cursor.execute("SELECT wigle FROM users WHERE username = ?", (username,))
row = cursor.fetchone()
api_key = row[0] if row else None

conn.close()

return api_key

with get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute("SELECT wigle FROM users WHERE username = ?", (username,))
row = cursor.fetchone()
return row[0] if row else None

def main():
if len(sys.argv) < 2:
print("Usage: python script_name.py <username>")
return
# Check for API key in creds.txt

print("Geolocate Wigle: Starting")
username = sys.argv[1]
api_key = get_key(username)
if api_key: # Ensure API key exists before attempting download
create_pwned_table()

if api_key:
new_networks, no_geolocation_networks, total_networks = populate_pwned_data(api_key)
print(f"Processed {total_networks} Networks in total. {new_networks} were new and {no_geolocation_networks} had no geolocation in the WiGLE API ")
print(f"Processed {total_networks} Networks in total. {new_networks} were new and {no_geolocation_networks} had no geolocation in the WiGLE API.")
else:
print("API key not found for the specified username. Set it up in the Settings")
print("API key not found for the specified username. Set it up in the Settings.")

print("✅ Geolocate Wigle: Finished")

if __name__ == '__main__':
Expand Down
32 changes: 16 additions & 16 deletions app/tools/manual_pot.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

timestamp = datetime.datetime.now().strftime("%d.%m.%Y %H:%M:%S")


cursor.execute('''CREATE TABLE IF NOT EXISTS "wpasec" (
"ap_mac" TEXT,
"sta_mac" TEXT,
Expand All @@ -35,27 +34,25 @@
csv_reader = csv.reader(file, delimiter=':')
for row in csv_reader:
total_networks += 1
# Check if the row contains exactly three colons
if len(row) == 4 or len(row) == 5: # Check if the row is format either like the legacy format or the new 22000 format
if len(row) == 5: # For new 22000 format which contains 5 field with a total of 4 ":"
line = ':'.join(row).split(':', 1)[1] #remove the first datafield
if len(row) == 4:
line = ':'.join(row) # For legacy format, keep the line as is
# Split the line into its components
ap_mac, sta_mac, ssid, password = line.split(':')
# Format MAC from abcdefhgij to ab:cd:ef:gh:ij
# Check if the row contains either four or five fields
if len(row) == 4 or len(row) == 5:
if len(row) == 5: # For new 22000 format which contains 5 fields
row = row[1:] # Remove the first field
# Extract components
ap_mac, sta_mac, ssid, password = row
# Format MAC addresses from abcdefghij to ab:cd:ef:gh:ij
ap_mac = ':'.join(ap_mac[i:i+2].upper() for i in range(0, len(ap_mac), 2))
sta_mac = ':'.join(sta_mac[i:i+2].upper() for i in range(0, len(sta_mac), 2))
cursor.execute('''SELECT COUNT(*) FROM "wpasec"
WHERE ap_mac=? AND sta_mac=? AND ssid=?''',
(ap_mac, sta_mac, ssid))
WHERE ap_mac=? AND sta_mac=? AND ssid=?''',
(ap_mac, sta_mac, ssid))
if cursor.fetchone()[0] == 0:
cursor.execute('''INSERT INTO "wpasec" (ap_mac, sta_mac, ssid, password, timestamp)
VALUES (?, ?, ?, ?, ?)''',
(ap_mac, sta_mac, ssid, password, timestamp))
VALUES (?, ?, ?, ?, ?)''',
(ap_mac, sta_mac, ssid, password, timestamp))
new_networks += 1
else:
print (f"Network {ssid} has already an entry, skipping")
print(f"Network {ssid} already has an entry, skipping")
duplicate_networks += 1
else:
print(f"Row was not in the expected format for a mode 22000 potfile: {row}")
Expand All @@ -64,4 +61,7 @@
conn.commit()
conn.close()

print("Successfully parsed all new PWNED Networks")
print(f"Successfully parsed all new PWNED Networks")
print(f"Total networks processed: {total_networks}")
print(f"New networks added: {new_networks}")
print(f"Duplicate networks skipped: {duplicate_networks}")
Loading

0 comments on commit 02fa869

Please sign in to comment.