Http to https

.gitignore

@@ -7,3 +7,4 @@
 /.mypy_cache
 /.ruff_cache
 /.cache
+/config.yaml

Makefile

@@ -11,24 +11,31 @@ PROXY_RUN = $(DOCKER_COMPOSE) run --rm proxy
 # Container management
 # --------------------
 
+.PHONY: config
+config: .env config.yaml
+
 # Create .env file to set the UID/GID for the docker containers to run as to the current user
 .env:
 	echo "DOCKER_LOCAL_USER=$(shell id -u):$(shell id -g)" >> .env
 
+# Create config file from config_dist_dev.yaml if it does not exist yet
+config.yaml:
+	cp config_dist_dev.yaml config.yaml
+
 .PHONY: docker-up
 docker-up: docker-build
 	$(DOCKER_COMPOSE) up
 
 .PHONY: docker-down
-docker-down: .env
+docker-down: config
 	$(DOCKER_COMPOSE) down --remove-orphans
 
 .PHONY: docker-purge
-docker-purge: .env
+docker-purge: config
 	$(DOCKER_COMPOSE) down --remove-orphans --volumes
 
 .PHONY: docker-build
-docker-build: .env
+docker-build: config
 	$(DOCKER_COMPOSE) build proxy
 
 

README.md

@@ -45,8 +45,15 @@ You can also use the interactive mode by
 mitmproxy -s addons.py
 ```
 
+If you want to have access to a config, you have to create a config.yaml. You can use the template in 
+`config_dist_dev.yaml`:
 
-Using docker, you just have to use `make` to start the `mitmdump` service.
+```shell
+cp config_dist_dev.yaml config.yaml
+```
+
+Using docker, you just have to use `make` to start the `mitmdump` service. The `config.yaml` is created with default
+values automatically.
 
 The HTTP proxy will be available at `http://localhost:8080`.
 

app/app.py

@@ -15,12 +15,15 @@
 from mitmproxy.http import HTTPFlow
 
 from app.base_converter import BaseConverter
+from app.config_helper import ConfigHelper
 
 
 class App:
     json_converters: Dict[str, List[BaseConverter]]
 
     def __init__(self):
+        self.config_helper = ConfigHelper()
+
         self.json_converters = {}
 
         # the following code is a converter autoloader. It dynamically adds all converters in ./converters.
@@ -41,6 +44,11 @@ def __init__(self):
                             self.json_converters[hostname] = []
                         self.json_converters[hostname].append(obj)
 
+    def request(self, flow: HTTPFlow):
+        if flow.request.host in self.config_helper.get('HTTP_TO_HTTPS_HOSTS', []):
+            flow.request.scheme = 'https'
+            flow.request.port = 443
+
     def response(self, flow: HTTPFlow):
         # if there is no converter for the requested host, don't do anything
         if flow.request.host not in self.json_converters:

app/config_helper.py

@@ -0,0 +1,26 @@
+"""
+MobiData BW Proxy
+Copyright (c) 2023, binary butterfly GmbH
+All rights reserved.
+"""
+
+from pathlib import Path
+from typing import Any
+
+from yaml import safe_load
+
+
+class ConfigHelper:
+    _config: dict
+
+    def __init__(self):
+        config_path = Path(Path(__file__).parents[1], 'config.yaml')
+
+        if config_path.exists():
+            with open(config_path) as config_file:
+                self._config = safe_load(config_file)
+        else:
+            self._config = {}
+
+    def get(self, key: str, default: Any) -> Any:
+        return self._config.get(key, default)

app/converters/gbfs_https_to_http.py

@@ -0,0 +1,30 @@
+"""
+MobiData BW Proxy
+Copyright (c) 2023, binary butterfly GmbH
+All rights reserved.
+"""
+
+from typing import Union
+
+from app.base_converter import BaseConverter
+
+
+class GbfsHttpsToHttpConverter(BaseConverter):
+    hostnames = ['gbfs.nextbike.net']
+
+    def convert(self, data: Union[dict, list], path: str) -> Union[dict, list]:
+        if not isinstance(data, dict) and not path.endswith('/gbfs.json'):
+            return data
+
+        if not isinstance(data, dict) or 'data' not in data or not isinstance(data['data'], dict):
+            return data
+
+        for language in data['data']:
+            if 'feeds' not in data['data'][language] or not isinstance(data['data'][language]['feeds'], list):
+                continue
+            for feed in data['data'][language]['feeds']:
+                if not isinstance(feed, dict) or 'url' not in feed or not isinstance(feed['url'], str):
+                    continue
+                feed['url'] = f'http{feed["url"][5:]}'
+
+        return data

config_dist_dev.yaml

@@ -0,0 +1,3 @@
+
+HTTP_TO_HTTPS_HOSTS:
+  - gbfs.nextbike.net

requirements-dev.txt

@@ -1,3 +1,4 @@
 ruff~=0.0.282
 black~=23.7.0
 mypy~=1.4.1
+types-pyyaml~=6.0.12.11

requirements.txt

@@ -1 +1 @@
-mitmproxy~=10.0.0
+mitmproxy~=10.0.0