Configure browsertrix proxies

.github/workflows/publish-helm-chart.yaml

@@ -23,6 +23,7 @@ jobs:
  run: |
  mkdir .chart-out
  helm package chart/ --destination .chart-out
+ helm package chart/proxies/ --destination .chart-out
 
  - name: Get Version
  run: |
@@ -49,7 +50,9 @@ jobs:
 
  See [the development guide](https://docs.browsertrix.com/deploy/) for more info how to deploy Browsertrix.
 
- files: .chart-out/browsertrix-v${{ env.version }}.tgz
+ files: |
+ .chart-out/browsertrix-v${{ env.version }}.tgz
+ .chart-out/btrix-proxies-0.1.0.tgz
  tag_name: v${{ env.version }}
  draft: true
  fail_on_unmatched_files: true

backend/btrixcloud/crawlconfigs.py

@@ -10,6 +10,7 @@
 import json
 import re
 import os
+import traceback
 from datetime import datetime
 from uuid import UUID, uuid4
 import urllib.parse
@@ -39,6 +40,8 @@
  CrawlConfigSearchValues,
  CrawlConfigUpdateResponse,
  CrawlConfigDeletedResponse,
+ CrawlerProxy,
+ CrawlerProxies,
 )
 from .utils import dt_now, slug_from_name
 
@@ -63,6 +66,8 @@
  "name",
 )
 
+DEFAULT_PROXY_ID: str | None = os.environ.get("DEFAULT_PROXY_ID")
+
 
 # ============================================================================
 class CrawlConfigOps:
@@ -125,6 +130,14 @@ def __init__(
  if "default" not in self.crawler_images_map:
  raise TypeError("The channel list must include a 'default' channel")
 
+ self._crawler_proxies_last_updated = None
+ self._crawler_proxies_map = None
+
+ if DEFAULT_PROXY_ID and DEFAULT_PROXY_ID not in self.get_crawler_proxies_map():
+ raise ValueError(
+ f"Configured proxies must include DEFAULT_PROXY_ID: {DEFAULT_PROXY_ID}"
+ )
+
  def set_crawl_ops(self, ops):
  """set crawl ops reference"""
  self.crawl_ops = ops
@@ -168,7 +181,9 @@ async def get_profile_filename(
  if not profileid:
  return ""
 
- profile_filename = await self.profiles.get_profile_storage_path(profileid, org)
+ profile_filename, _ = await self.profiles.get_profile_storage_path_and_proxy(
+ profileid, org
+ )
  if not profile_filename:
  raise HTTPException(status_code=400, detail="invalid_profile_id")
 
@@ -195,6 +210,11 @@ async def add_crawl_config(
  if profileid:
  await self.profiles.get_profile(profileid, org)
 
+ # ensure proxyId is valid and available for org
+ if config_in.proxyId:
+ if not self.can_org_use_proxy(org, config_in.proxyId):
+ raise HTTPException(status_code=404, detail="proxy_not_found")
+
  now = dt_now()
  crawlconfig = CrawlConfig(
  id=uuid4(),
@@ -218,6 +238,7 @@ async def add_crawl_config(
  profileid=profileid,
  crawlerChannel=config_in.crawlerChannel,
  crawlFilenameTemplate=config_in.crawlFilenameTemplate,
+ proxyId=config_in.proxyId,
  )
 
  if config_in.runNow:
@@ -331,6 +352,8 @@ async def update_crawl_config(
  and ((not update.profileid) != (not orig_crawl_config.profileid))
  )
 
+ changed = changed or (orig_crawl_config.proxyId != update.proxyId)
+
  metadata_changed = self.check_attr_changed(orig_crawl_config, update, "name")
  metadata_changed = metadata_changed or self.check_attr_changed(
  orig_crawl_config, update, "description"
@@ -829,6 +852,9 @@ async def run_now_internal(
  if await self.get_running_crawl(crawlconfig.id):
  raise HTTPException(status_code=400, detail="crawl_already_running")
 
+ if crawlconfig.proxyId and not self.can_org_use_proxy(org, crawlconfig.proxyId):
+ raise HTTPException(status_code=404, detail="proxy_not_found")
+
  profile_filename = await self.get_profile_filename(crawlconfig.profileid, org)
  storage_filename = (
  crawlconfig.crawlFilenameTemplate or self.default_filename_template
@@ -848,6 +874,7 @@ async def run_now_internal(
 
  except Exception as exc:
  # pylint: disable=raise-missing-from
+ print(traceback.format_exc())
  raise HTTPException(status_code=500, detail=f"Error starting crawl: {exc}")
 
  async def set_config_current_crawl_info(
@@ -897,6 +924,68 @@ def get_channel_crawler_image(
  """Get crawler image name by id"""
  return self.crawler_images_map.get(crawler_channel or "")
 
+ def get_crawler_proxies_map(self) -> dict[str, CrawlerProxy]:
+ """Load CrawlerProxy mapping from config"""
+ proxies_last_update_path = os.environ["CRAWLER_PROXIES_LAST_UPDATE"]
+
+ if not os.path.isfile(proxies_last_update_path):
+ return {}
+
+ # return cached data, when last_update timestamp hasn't changed
+ if self._crawler_proxies_last_updated and self._crawler_proxies_map:
+ with open(proxies_last_update_path, encoding="utf-8") as fh:
+ proxies_last_update = int(fh.read().strip())
+ if proxies_last_update == self._crawler_proxies_last_updated:
+ return self._crawler_proxies_map
+ self._crawler_proxies_last_updated = proxies_last_update
+
+ crawler_proxies_map: dict[str, CrawlerProxy] = {}
+ with open(os.environ["CRAWLER_PROXIES_JSON"], encoding="utf-8") as fh:
+ proxy_list = json.loads(fh.read())
+ for proxy_data in proxy_list:
+ proxy = CrawlerProxy(
+ id=proxy_data["id"],
+ label=proxy_data["label"],
+ description=proxy_data.get("description", ""),
+ country_code=proxy_data.get("country_code", ""),
+ url=proxy_data["url"],
+ has_host_public_key=bool(proxy_data.get("ssh_host_public_key")),
+ has_private_key=bool(proxy_data.get("ssh_private_key")),
+ shared=proxy_data.get("shared", False)
+ or proxy_data["id"] == DEFAULT_PROXY_ID,
+ )
+
+ crawler_proxies_map[proxy.id] = proxy
+
+ self._crawler_proxies_map = crawler_proxies_map
+ return self._crawler_proxies_map
+
+ def get_crawler_proxies(self):
+ """Get CrawlerProxy configuration"""
+ return CrawlerProxies(
+ default_proxy_id=DEFAULT_PROXY_ID,
+ servers=list(self.get_crawler_proxies_map().values()),
+ )
+
+ def get_crawler_proxy(self, proxy_id: str) -> Optional[CrawlerProxy]:
+ """Get crawlerProxy by id"""
+ return self.get_crawler_proxies_map().get(proxy_id)
+
+ def can_org_use_proxy(self, org: Organization, proxy: CrawlerProxy | str) -> bool:
+ """Checks if org is able to use proxy"""
+
+ if isinstance(proxy, str):
+ _proxy = self.get_crawler_proxy(proxy)
+ else:
+ _proxy = proxy
+
+ if _proxy is None:
+ return False
+
+ return (
+ _proxy.shared and org.allowSharedProxies
+ ) or _proxy.id in org.allowedProxies
+
  def get_warc_prefix(self, org: Organization, crawlconfig: CrawlConfig) -> str:
  """Generate WARC prefix slug from org slug, name or url
  if no name is provided, hostname is used from url, otherwise
@@ -983,6 +1072,7 @@ async def stats_recompute_all(crawl_configs, crawls, cid: UUID):
 # ============================================================================
 # pylint: disable=redefined-builtin,invalid-name,too-many-locals,too-many-arguments
 def init_crawl_config_api(
+ app,
  dbclient,
  mdb,
  user_dep,
@@ -1060,6 +1150,28 @@ async def get_crawler_channels(
  ):
  return ops.crawler_channels
 
+ @router.get("/crawler-proxies", response_model=CrawlerProxies)
+ async def get_crawler_proxies(
+ org: Organization = Depends(org_crawl_dep),
+ ):
+ return CrawlerProxies(
+ default_proxy_id=DEFAULT_PROXY_ID,
+ servers=[
+ proxy
+ for proxy in ops.get_crawler_proxies_map().values()
+ if ops.can_org_use_proxy(org, proxy)
+ ],
+ )
+
+ @app.get("/orgs/all/crawlconfigs/crawler-proxies", response_model=CrawlerProxies)
+ async def get_all_crawler_proxies(
+ user: User = Depends(user_dep),
+ ):
+ if not user.is_superuser:
+ raise HTTPException(status_code=403, detail="Not Allowed")
+
+ return ops.get_crawler_proxies()
+
  @router.get("/{cid}/seeds", response_model=PaginatedSeedResponse)
  async def get_crawl_config_seeds(
  cid: UUID,

backend/btrixcloud/crawlmanager.py

@@ -1,7 +1,6 @@
 """ shared crawl manager implementation """
 
 import os
-import asyncio
 import secrets
 
 from typing import Optional, Dict
@@ -16,13 +15,12 @@
 
 
 # ============================================================================
-class CrawlManager(K8sAPI):
- """abstract crawl manager"""
+DEFAULT_PROXY_ID: str = os.environ.get("DEFAULT_PROXY_ID", "")
 
- def __init__(self):
- super().__init__()
 
- self.loop = asyncio.get_running_loop()
+# ============================================================================
+class CrawlManager(K8sAPI):
+ """abstract crawl manager"""
 
  # pylint: disable=too-many-arguments
  async def run_profile_browser(
@@ -34,6 +32,7 @@ async def run_profile_browser(
  crawler_image: str,
  baseprofile: str = "",
  profile_filename: str = "",
+ proxy_id: str = "",
  ) -> str:
  """run browser for profile creation"""
 
@@ -55,6 +54,7 @@ async def run_profile_browser(
  "vnc_password": secrets.token_hex(16),
  "expire_time": date_to_str(dt_now() + timedelta(seconds=30)),
  "crawler_image": crawler_image,
+ "proxy_id": proxy_id or DEFAULT_PROXY_ID,
  }
 
  data = self.templates.env.get_template("profile_job.yaml").render(params)
@@ -138,6 +138,7 @@ async def create_crawl_job(
  warc_prefix=warc_prefix,
  storage_filename=storage_filename,
  profile_filename=profile_filename,
+ proxy_id=crawlconfig.proxyId or DEFAULT_PROXY_ID,
  )
 
  async def create_qa_crawl_job(

backend/btrixcloud/crawls.py

@@ -379,6 +379,7 @@ async def add_new_crawl(
  tags=crawlconfig.tags,
  name=crawlconfig.name,
  crawlerChannel=crawlconfig.crawlerChannel,
+ proxyId=crawlconfig.proxyId,
  image=image,
  )
 

backend/btrixcloud/k8sapi.py

@@ -2,8 +2,8 @@
 
 import os
 import traceback
-
 from typing import Optional
+
 import yaml
 
 from kubernetes_asyncio import client, config
@@ -93,6 +93,7 @@ def new_crawl_job_yaml(
  storage_filename: str = "",
  profile_filename: str = "",
  qa_source: str = "",
+ proxy_id: str = "",
  ):
  """load job template from yaml"""
  if not crawl_id:
@@ -115,6 +116,7 @@ def new_crawl_job_yaml(
  "storage_filename": storage_filename,
  "profile_filename": profile_filename,
  "qa_source": qa_source,
+ "proxy_id": proxy_id,
  }
 
  data = self.templates.env.get_template("crawl_job.yaml").render(params)
@@ -136,6 +138,7 @@ async def new_crawl_job(
  storage_filename: str = "",
  profile_filename: str = "",
  qa_source: str = "",
+ proxy_id: str = "",
  ) -> str:
  """load and init crawl job via k8s api"""
  crawl_id, data = self.new_crawl_job_yaml(
@@ -153,6 +156,7 @@ async def new_crawl_job(
  storage_filename=storage_filename,
  profile_filename=profile_filename,
  qa_source=qa_source,
+ proxy_id=proxy_id,
  )
 
  # create job directly

backend/btrixcloud/main.py

@@ -205,6 +205,7 @@ def main() -> None:
  )
 
  crawl_config_ops = init_crawl_config_api(
+ app,
  dbclient,
  mdb,
  current_active_user,