Pre-Multithreading

Signed-off-by: TheBoatyMcBoatFace <[email protected]>

app/database/postgres/queries/clothe_domains.sql

@@ -1,2 +1,6 @@
 -- app/database/postgres/queries/clothe_domains.sql
 -- Creates url entry
+UPDATE targets.domains d
+SET home_url = :home_url
+WHERE id = :domain_id
+RETURNING id;

app/database/postgres/queries/get_naked_domains.sql

@@ -1,8 +1,10 @@
+-- app/database/postgres/queries/get_naked_domains.sql
 SELECT
   id AS "domain_id",
   "domain"
 FROM targets.domains d
-WHERE home_url IS NULL
+WHERE (home_url IS NULL OR home_url = '')
   AND active = TRUE
   AND "valid" = TRUE
 LIMIT 1;
+

app/database/postgres/run_query.py

@@ -14,26 +14,20 @@ def run_query(query_name, vars=None):
     with open(query_file) as file:
         sql_content = file.read()
 
-    if vars:
-        formatted_sql_content = text(sql_content) % tuple(vars)
-    else:
-        formatted_sql_content = text(sql_content)
-
     logger.info(f"Running query: {query_name}")
 
     session = conn()
 
     try:
-        result = session.execute(formatted_sql_content)
+        result = session.execute(text(sql_content), vars)
+        logger.debug(f'Formatted SQL to Run:\n %s', sql_content)
+
         session.commit()
 
-        if re.match(r"(SELECT|UPDATE)", sql_content, re.IGNORECASE):
-            rows = result.fetchall()
-            logger.debug(f"Result rows: {rows}")
-            return rows
-        else:  # For INSERT, DELETE or any other query type
-            logger.info(f"Affected rows: {result.rowcount}")
-            return {"affected_rows": result.rowcount}
+        rows = result.fetchall()
+        logger.debug(f"Result rows: {rows}")
+
+        return rows
 
     except SQLAlchemyError as e:
         session.rollback()

app/processes/__init__.py

@@ -1,11 +1,26 @@
 # app/processes/__init__.py
+import time
+from multiprocessing import Process
+from app import logger
 
-# from .axe import get_axes, execute_axes
-# from .preprocess_tests import preprocess_data
-
+# Process Imports
 from .naked_urls import find_nakies
-from app import logger
+
+def process_loop(process_func, sleep_time):
+    while True:
+        if not process_func():  # If there is no data to process
+            time.sleep(sleep_time)  # Wait for the specified amount of time
+
 
 def start_processes():
     logger.info('Starting processes...')
-    find_nakies()
+
+    # Functions with their sleep times
+    processes = [
+        (find_nakies, 30)
+        #(fix_axe, 60)
+    ]
+
+    for process_func, sleep_time in processes:
+        process = Process(target=process_loop, args=(process_func, sleep_time))
+        process.start()

app/processes/naked_urls.py

@@ -0,0 +1,53 @@
+from app import logger
+from app.database.postgres.run_query import run_query
+import requests
+
+def find_nakies():
+    logger.info('Starting to find naked domains...')
+    query_name = "get_naked_domains"
+    result = run_query(query_name)
+
+    logger.debug('SQL result: %s', result)
+
+    if result:
+        try:
+            domain_id, domain = result[0]
+        except IndexError:
+            logger.error('No rows were returned from the SQL query.')
+            return False  # No data to process
+
+        home_url = get_home_url(domain)
+        record_home_url(domain_id, home_url)
+
+        if home_url == "BADDIE":
+            logger.debug(f'We got a BADDIE for %s', domain)
+        else:
+            logger.debug(f'%s\'s home url is: %s', domain, home_url)
+
+        return True  # There is data to process
+
+    else:
+        logger.info('No naked domains found.')
+        return False  # No data to process
+
+
+def get_home_url(domain):
+    logger.debug(f'Getting home url for %s', domain)
+
+    try:
+        response = requests.get(f'http://{domain}', timeout=5, allow_redirects=True)
+
+        if response.status_code == 200:
+            return response.url
+        else:
+            return "BADDIE"
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Error while getting home URL for {domain}: {str(e)}")
+        return "BADDIE"
+
+def record_home_url(domain_id, home_url):
+    logger.debug('Fixing home_url for domain_id: %s', domain_id)
+    query_name = "clothe_domains"
+    variables = {"domain_id": domain_id, "home_url": home_url}
+    result = run_query(query_name, variables)
+