Skip to content

Commit

Permalink
fixed wappalyzer db
Browse files Browse the repository at this point in the history
  • Loading branch information
smaury committed Nov 7, 2021
1 parent 268e8b0 commit 0aacf16
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 31 deletions.
2 changes: 1 addition & 1 deletion webtech/__version__.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
# DON'T EDIT THIS FILE
__version__ = "1.2.12"
__version__ = "1.3"
76 changes: 46 additions & 30 deletions webtech/database.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os.path
from os import unlink
import time
from .__burp__ import BURP
from .utils import user_data_dir
from string import ascii_lowercase
import json

if not BURP:
try:
Expand All @@ -21,7 +24,7 @@
os.mkdir(DATA_DIR)
DATABASE_FILE = os.path.join(DATA_DIR, "webtech.json")
WAPPALYZER_DATABASE_FILE = os.path.join(DATA_DIR, "apps.json")
WAPPALYZER_DATABASE_URL = "https://raw.githubusercontent.com/AliasIO/wappalyzer/master/src/technologies.json"
WAPPALYZER_DATABASE_URL_BASE = "https://raw.githubusercontent.com/AliasIO/wappalyzer/master/src/technologies/"
WEBTECH_DATABASE_URL = "https://raw.githubusercontent.com/ShielderSec/webtech/master/webtech/webtech.json"
DAYS = 60 * 60 * 24

Expand All @@ -30,54 +33,67 @@ def download_database_file(url, target_file):
"""
Download the database file from the WAPPPALIZER repository
"""
print("Updating database...")
print("Downloading database: {}".format(url))
response = urlopen(url)
with open(target_file, 'wb') as out_file:
out_file.write(response.read())
print("Database updated successfully!")


def download(webfile, dbfile, name, force=False):
def download():
"""
Check if outdated and download file
"""
try:
download_database_file(WEBTECH_DATABASE_URL, DATABASE_FILE)
except:
pass
with open(WAPPALYZER_DATABASE_FILE, 'w') as f:
json.dump({"apps":{}}, f)

for c in ascii_lowercase + "_":
try:
download_database_file("{}{}.json".format(WAPPALYZER_DATABASE_URL_BASE,c), os.path.join(DATA_DIR,"temp.json"))
merge_partial_wappalyzer_database()
unlink(os.path.join(DATA_DIR,"temp.json"))
except URLError as e:
print("The Wappalyzer database seems offline. Report this issue to: https://github.com/ShielderSec/webtech/")
pass


def update_database(args=None, force=False):
"""
Update the database if it's not present or too old
"""
now = int(time.time())
if not os.path.isfile(dbfile):
print("{} Database file not present.".format(name))
download_database_file(webfile, dbfile)
# set timestamp in filename
if not os.path.isfile(WAPPALYZER_DATABASE_FILE):
print("Database file not present.")
download()
else:
last_update = int(os.path.getmtime(dbfile))
last_update = int(os.path.getmtime(WAPPALYZER_DATABASE_FILE))
if last_update < now - 30 * DAYS or force:
if force:
print("Force update of {} Database file".format(name))
print("Force update of Database file")
else:
print("{} Database file is older than 30 days.".format(name))
os.remove(dbfile)
download_database_file(webfile, dbfile)
print("Database file is older than 30 days.")
unlink(WAPPALYZER_DATABASE_FILE)
download()



def update_database(args=None, force=False):
def merge_partial_wappalyzer_database():
"""
Update the database if it's not present or too old
This helper function merges a partial wappalyzer db with the other ones.
"""
try:
download(WAPPALYZER_DATABASE_URL, WAPPALYZER_DATABASE_FILE, "Wappalyzer", force=force)
except URLError as e:
print("The Wappalyzer database seems offline. Report this issue to: https://github.com/ShielderSec/webtech/")
pass

try:
download(WEBTECH_DATABASE_URL, DATABASE_FILE, "WebTech", force=force)
return True
except URLError as e:
print("Unable to update database, check your internet connection and Github.com availability.")
return False


with open(WAPPALYZER_DATABASE_FILE, 'r+') as f1:
with open(os.path.join(DATA_DIR,"temp.json")) as f2:
current = json.load(f1)
temp = {"apps": json.load(f2)}
f1.seek(0)
json.dump(merge_databases(current, temp),f1)

def merge_databases(db1, db2):
"""
This helper function merge elements from two databases without overrding its elements
This helper function merges elements from two databases without overrding its elements
This function is not generic and *follow the Wappalyzer db scheme*
"""
# Wappalyzer DB format must have an apps/technologies object
Expand Down

0 comments on commit 0aacf16

Please sign in to comment.