From 8120c0e88a095f8ee90f7ca89e6336ba5a9e8646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Wed, 9 Oct 2024 15:57:50 -0700 Subject: [PATCH 01/27] Initial implemention of a push av server --- src/python_testing/TC_PAVS_1_0.py | 158 +++++ src/python_testing/push_av_server.py | 1 + src/tools/push_av_server/.flake8 | 2 + src/tools/push_av_server/README.md | 39 ++ .../push_av_server/generate_cmaf_content.sh | 126 ++++ src/tools/push_av_server/index.html | 87 +++ src/tools/push_av_server/requirements.txt | 4 + src/tools/push_av_server/server.py | 585 ++++++++++++++++++ 8 files changed, 1002 insertions(+) create mode 100644 src/python_testing/TC_PAVS_1_0.py create mode 120000 src/python_testing/push_av_server.py create mode 100644 src/tools/push_av_server/.flake8 create mode 100644 src/tools/push_av_server/README.md create mode 100755 src/tools/push_av_server/generate_cmaf_content.sh create mode 100644 src/tools/push_av_server/index.html create mode 100644 src/tools/push_av_server/requirements.txt create mode 100644 src/tools/push_av_server/server.py diff --git a/src/python_testing/TC_PAVS_1_0.py b/src/python_testing/TC_PAVS_1_0.py new file mode 100644 index 00000000000000..288eb3626e34a0 --- /dev/null +++ b/src/python_testing/TC_PAVS_1_0.py @@ -0,0 +1,158 @@ +import logging +import time +import http.server +import ssl +import threading +import cryptography +import tempfile +import os.path +import datetime +from functools import partial +import shutil +# What's the stands of the Matter project on depending on requests ? +# TODO Actually do we really need requests once we have matter cameras in place? +import requests +import subprocess +import json +import socket +import sys + +import push_av_server + +import requests.adapters +from zeroconf import ServiceInfo, Zeroconf +from urllib.parse import urlparse + +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +import chip.clusters as Clusters +from chip.clusters import ClusterObjects as ClusterObjects +from matter_testing_support import (ClusterAttributeChangeAccumulator, MatterBaseTest, TestStep, default_matter_test_main, + async_test_body) +from mobly import asserts +from test_plan_support import commission_if_required, if_feature_supported, read_attribute, verify_success + + +class TC_PAVS_1_0(MatterBaseTest): + """ + NOTE: this class is only a guide to understand what APIs I'd need to integrate in the push av server + for a better integration. It is not designed to be merged nor does it actually run. + """ + + def steps_TC_PAVS_1_0(self): + return [TestStep(1, "Commissioning, already done", is_commissioning=True), + TestStep(2, "Install CA onto the device"), + TestStep(3, "Obtain device CSR, generate cert, provision cert onto device"), + TestStep(4, "Create media streams"), + TestStep(5, "Allocate push transport"), + TestStep(6, "Trigger a recording"), + TestStep(7, "Deallocate transport") + ] + + @async_test_body + async def test_TC_PAVS_1_0(self): + srv = push_av_server.start("localhost", 1234) + srv.run_in_thread() + + # commissioning - already done + self.step(1) + + self.step(2) + # Access CA cert via the push_av_server package. + push_av_server.device_hierarchy.root_cert + # read TLSCertificateManagament attributes to validate state + # Send the TLSCertificateManagament.ProvisionRootCertificate command + # Assert we got a response that contains a CA id + # read TLSCertificateManagament attributes to validate state + + self.step(3) + + self.step("3b") + # Generate nonce + # send TLSCertificateManagement.TLSClientCSR, receive TLSClientCSRResponse + push_av_server.device_hierarchy.gen_cert(name, csr) + # send ProvisionClientCertificate, receive ProvisionClientCertificateResponse + + self.step(4) + # (note: assum this step is a requirement and not the focus of these TCs) + # send VideoStreamAllocate, receive VideoStreamAllocateResponse + # StreamType: StreamTypeEnum.Recording + # VideoCodec: VideoCodecEnum.H264 (HEVC, VVC, AV1 are all optionals) + # MinFrameRate: 0 + # MaxFrameRate: 60 + # MinResolution: 0 + # MaxResolution: 4k + # MinBitRate: 0 + # MaxBitRate: inf + # MinFragmentLen: 0 + # MaxFragmentLen: info + # send AudioStreamAllocate, receive AudioStreamAllocateResponse + # StreamType: StreamTypeEnum.Recording + # AudioCodec: AudioCodecEnum.OPUS (AAC-LC is optional) + # ChannelCount: 1 (note: or 2? what's the requirements that works for most cameras) + # SampleRate: TBD (48, 32, 16khz) + # BitRate: TBD + # BitDepth: TBD + + self.step(5) + # send AllocatePushTransport, receive AllocatePushTransportResponse + # PushAVStreamTransportOptionsStruct: + # video stream id: from step 4 + # audio stream id: from step 4 + # tls endpoint id: from step 3b + # url: local dns + known path from step 2 + # triggerOptions: PushAVStreamTransportMotionTriggerTimeControlStruct + # InitialDuration: default + # AugmentationDuration: default + # MaxDuration: default + # BlindDuration: default + # (note: are we testing this in this test plan or in webrtc?) + # containerFormat: PushAVStreamTransportContainerFormatEnum.CMAF (only one at the time) + # ingestMethod: PushAVStreamTransportIngestMethodEnum.CMAFIngest (only one at the time) + # containerOptions: PushAVStreamTransportContainerOptionsStruct + # ContainerType: PushAVStreamTransportContainerFormatEnum.CMAF (only one at the time) + # CMAFContainerOptions: PushAVStreamTransportCMAFContainerOptionsStruct + # ChunkDuration: default + # CENCKey: null. (note: do we test this in the harnes or do we not?) + # metadataOptions: PushAVStreamTransportMetadataOptionsStruct + # Multiplexing: PushAVStreamTransportStreamMultiplexingEnum.Interleaved + # IncludeMotionsZones: false + # EnablePrivacySensitive: false + # expiryTime: null? Not entirely sure how to test this one yet. + + # find stream config and assert + # modify stream + # find stream config and assert + + # set transport status + # find stream config and assert + # reset transport status + # find stream config and assert + + self.step(6) + # subscribe to PushTransport events (note: forgot if it's required or not, I think it is) + # send ManuallyTriggerTransport + # ConnectionId: from step 5 + # Action: PushAVStreamTransport_ActionEnum + # ActivationReason: PushAVStreamTransportTriggerActivationReasonEnum + # MinDuration: 5 + # read + # listen for PushTransportStart and PushTransportEnd event + # wait for start event, validate conn id and options + # wait for end event, validate conn id and options + + # Check metadata of stream sent to our web server + # ffmpeg convert the cmaf tracks into something more easily read by viewers + # manual step to inspect the video + + self.step(7) + # TBD. deallocation logic + + srv.stop() + + +if __name__ == "__main__": + default_matter_test_main() diff --git a/src/python_testing/push_av_server.py b/src/python_testing/push_av_server.py new file mode 120000 index 00000000000000..c808a5e73628fa --- /dev/null +++ b/src/python_testing/push_av_server.py @@ -0,0 +1 @@ +../tools/push_av_server/server.py \ No newline at end of file diff --git a/src/tools/push_av_server/.flake8 b/src/tools/push_av_server/.flake8 new file mode 100644 index 00000000000000..51b50a0465baf3 --- /dev/null +++ b/src/tools/push_av_server/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 100 \ No newline at end of file diff --git a/src/tools/push_av_server/README.md b/src/tools/push_av_server/README.md new file mode 100644 index 00000000000000..e717201138b645 --- /dev/null +++ b/src/tools/push_av_server/README.md @@ -0,0 +1,39 @@ +# Push AV Server + +This tool provide a web server that can be used to implement Matter cameras. The +server does not go out of its way to provide validation of the media ingested +(run the test harness to do so), but it does offer as much visibility as +possible on what the ingest source is sending to the server. + +## Example + +Here is an example of an interaction with the push AV server tool. + +```sh +$ python server.py --working-directory ~/.pavstest + + +# First let's create a device key and certificate. +# The response will provide information as to where the key and certificate are located. +$ curl --cacert ~/.pavstest/certs/server/root.pem -XPOST https://localhost:1234/certs/dev/keypair + +# Now that we have a device identity, we can create a stream +$ curl --cacert ~/.pavstest/certs/server/root.pem --cert ~/.pavstest/certs/device/dev.pem --key ~/.pavstest/certs/device/dev.key -XPOST https://localhost:1234/streams + +# And now that we have access to our stream_id, we can build the publishing endpoint for +# any CMAF ingest flow we have. The example below assuming a stream id of "1". +$ export PUBLISHING_ENDPOINT=https://localhost:1234/streams/1 + +# The tool also contains a script to generate arbitrary CMAF content. +# This may be useful to implementers of a publish endpoint. +# This tool makes use of the previously created PUBLISHING_ENDPOINT environment variable. +# TODO Handle non-hardcoded client certificate +$ ./generate_cmaf_content.sh + +# You can also list all streams and their associated files +$ curl -XGET --cacert ~/.pavstest/certs/server/root.pem https://localhost:1234/streams + +# Get detailed information about the uploaded media file. +# This correspond to the ffprobe tool output +$ curl --cacert ~/.pavstest/certs/server/root.pem -XGET 'https://localhost:1234/streams/probe/1/cmaf/example.str/Switching(video)/video-720p.cmfv' +``` diff --git a/src/tools/push_av_server/generate_cmaf_content.sh b/src/tools/push_av_server/generate_cmaf_content.sh new file mode 100755 index 00000000000000..194f4029326a26 --- /dev/null +++ b/src/tools/push_av_server/generate_cmaf_content.sh @@ -0,0 +1,126 @@ +# source https://github.com/nagare-media/ingest/blob/main/scripts/tasks/run-cmaf-long-upload-ffmpeg +# Copyright 2022-2024 The nagare media authors under Apache 2.0 + +PUBLISHING_ENDPOINT=${PUBLISHING_ENDPOINT:-https://localhost:1234/stream/1} + +# TODO Handle dynamic value for those three variables +HTTP_OPTS=ca_file=~/.pavstest/certs/server/root.pem,cert_file=~/.pavstest/certs/device/dev.pem,key_file=~/.pavstest/certs/device/dev.key + +ffmpeg -hide_banner \ + -re -f lavfi -i " + testsrc2=size=1280x720:rate=25, + drawbox=x=0:y=0:w=700:h=50:c=black@.6:t=fill, + drawtext=x= 5:y=5:fontsize=54:fontcolor=white:text='%{pts\:gmtime\:$(date +%s)\:%Y-%m-%d}', + drawtext=x=345:y=5:fontsize=54:fontcolor=white:timecode='$(date -u '+%H\:%M\:%S')\:00':rate=25:tc24hmax=1, + setparams=field_mode=prog:range=tv:color_primaries=bt709:color_trc=bt709:colorspace=bt709, + format=yuv420p" \ + -re -f lavfi -i " + sine=f=1000:r=48000:samples_per_frame='st(0,mod(n,5)); 1602-not(not(eq(ld(0),1)+eq(ld(0),3)))'" \ + -shortest \ + -fflags genpts \ + \ + -filter_complex " + [0:v]drawtext=x=(w-text_w)-5:y=5:fontsize=54:fontcolor=white:text='720p':box=1:boxcolor=black@.6:boxborderw=5[v720p]; + [0:v]drawtext=x=(w-text_w)-5:y=5:fontsize=54:fontcolor=white:text='360p':box=1:boxcolor=black@.6:boxborderw=5,scale=640x360[v360p] + " \ + \ + -map [v720p] \ + -c:v libx264 \ + -preset:v veryfast \ + -tune zerolatency \ + -profile:v main \ + -crf:v 23 -bufsize:v:0 2250k -maxrate:v 2500k \ + -g:v 100000 -keyint_min:v 50000 -force_key_frames:v "expr:gte(t,n_forced*2)" \ + -x264opts no-open-gop=1 \ + -bf 2 -b_strategy 2 -refs 1 \ + -rc-lookahead 24 \ + -export_side_data prft \ + -field_order progressive -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv \ + -pix_fmt yuv420p \ + -f mp4 \ + -frag_duration "$((1 * 1000 * 1000))" \ + -min_frag_duration "$((1 * 1000 * 1000))" \ + -write_prft wallclock \ + -use_editlist 0 \ + -movflags "+cmaf+dash+delay_moov+skip_sidx+skip_trailer+frag_custom" \ + \ + -method PUT \ + -multiple_requests 1 \ + -chunked_post 1 \ + -send_expect_100 1 \ + -headers "DASH-IF-Ingest: 1.1" \ + -headers "Host: localhost:8080" \ + -content_type "" \ + -icy 0 \ + -rw_timeout "$((200 * 1000 * 1000))" \ + -reconnect 1 \ + -reconnect_at_eof 1 \ + -reconnect_on_network_error 1 \ + -reconnect_on_http_error 4xx,5xx \ + -reconnect_delay_max 2 \ + -http_opts $HTTP_OPTS \ + "$PUBLISHING_ENDPOINT/cmaf/example.str/Switching(video)/video-720p.cmfv" \ + \ + -map [v360p] \ + -c:v libx264 \ + -preset:v veryfast \ + -tune zerolatency \ + -profile:v main \ + -crf:v 23 -bufsize:v:0 2250k -maxrate:v 2500k \ + -g:v 100000 -keyint_min:v 50000 -force_key_frames:v "expr:gte(t,n_forced*2)" \ + -x264opts no-open-gop=1 \ + -bf 2 -b_strategy 2 -refs 1 \ + -rc-lookahead 24 \ + -export_side_data prft \ + -field_order progressive -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv \ + -pix_fmt yuv420p \ + -f mp4 \ + -frag_duration "$((1 * 1000 * 1000))" \ + -min_frag_duration "$((1 * 1000 * 1000))" \ + -write_prft wallclock \ + -use_editlist 0 \ + -movflags "+cmaf+dash+delay_moov+skip_sidx+skip_trailer+frag_custom" \ + \ + -method PUT \ + -multiple_requests 1 \ + -chunked_post 1 \ + -send_expect_100 1 \ + -headers "DASH-IF-Ingest: 1.1" \ + -headers "Host: localhost:8080" \ + -content_type "" \ + -icy 0 \ + -rw_timeout "$((200 * 1000 * 1000))" \ + -reconnect 1 \ + -reconnect_at_eof 1 \ + -reconnect_on_network_error 1 \ + -reconnect_on_http_error 4xx,5xx \ + -reconnect_delay_max 2 \ + -http_opts $HTTP_OPTS \ + "$PUBLISHING_ENDPOINT/cmaf/example.str/Switching(video)/video-360p.cmfv" \ + \ + -map 1:a \ + -c:a aac \ + -b:a 64k \ + -f mp4 \ + -frag_duration "$((1 * 1000 * 1000))" \ + -min_frag_duration "$((1 * 1000 * 1000))" \ + -write_prft wallclock \ + -use_editlist 0 \ + -movflags "+cmaf+dash+delay_moov+skip_sidx+skip_trailer+frag_custom" \ + \ + -method PUT \ + -multiple_requests 1 \ + -chunked_post 1 \ + -send_expect_100 1 \ + -headers "DASH-IF-Ingest: 1.1" \ + -headers "Host: localhost:8080" \ + -content_type "" \ + -icy 0 \ + -rw_timeout "$((200 * 1000 * 1000))" \ + -reconnect 1 \ + -reconnect_at_eof 1 \ + -reconnect_on_network_error 1 \ + -reconnect_on_http_error 4xx,5xx \ + -reconnect_delay_max 2 \ + -http_opts $HTTP_OPTS \ + "$PUBLISHING_ENDPOINT/cmaf/example.str/Switching(audio)/audio-64k.cmfa" diff --git a/src/tools/push_av_server/index.html b/src/tools/push_av_server/index.html new file mode 100644 index 00000000000000..1e670229700f08 --- /dev/null +++ b/src/tools/push_av_server/index.html @@ -0,0 +1,87 @@ + + + + + Push AV Ref Server + + + +

AV Push Server

+

Streams

+
+

Certificates

+
+

Server certificates

+
+

Device certificates

+
+
+ + \ No newline at end of file diff --git a/src/tools/push_av_server/requirements.txt b/src/tools/push_av_server/requirements.txt new file mode 100644 index 00000000000000..6269524a672778 --- /dev/null +++ b/src/tools/push_av_server/requirements.txt @@ -0,0 +1,4 @@ +zeroconf +cryptography +uvicorn +fastapi \ No newline at end of file diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py new file mode 100644 index 00000000000000..d61ce0461a3562 --- /dev/null +++ b/src/tools/push_av_server/server.py @@ -0,0 +1,585 @@ +# TODO Move this one src/tools/ instead of having it here +import logging +import ssl +from typing import Optional, Union +import tempfile +import os.path +import datetime + +# What's the stands of the Matter project on depending on requests ? +# TODO Actually do we really need requests once we have matter cameras in place? +import argparse +import pathlib +from pathlib import Path +import random +import string +import json +import socket +import sys +import subprocess + +from zeroconf import ServiceInfo, Zeroconf + +import uvicorn +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.asymmetric.types import ( + CERTIFICATE_PUBLIC_KEY_TYPES, + CERTIFICATE_PRIVATE_KEY_TYPES, +) + + +from fastapi import FastAPI, Request, HTTPException, Response +from fastapi.responses import HTMLResponse +from pydantic import BaseModel + +# Monkey patch uvicorn to make the underlying transport available to us. +# That will let us access the ssl context and get the client certificate information. +from uvicorn.protocols.http.h11_impl import H11Protocol + +http_tools_protocol_old__should_upgrade = H11Protocol._should_upgrade + + +def http_tools_protocol_new__should_upgrade(self): + http_tools_protocol_old__should_upgrade(self) + self.scope["transport"] = self.transport + + +H11Protocol._should_upgrade = http_tools_protocol_new__should_upgrade + +# End monkey patch + + +class WorkingDirectory: + """ + Collection of utilities to add convention to the files used by this program. + """ + + tmp = None + + def __init__(self, directory: Optional[str] = None) -> None: + + if directory is None: + self.tmp = tempfile.TemporaryDirectory(prefix="TC_PAVS_1_0") + else: + d = pathlib.Path(directory) + d.mkdir(parents=True, exist_ok=True) + self.directory = d + + def __enter__(self): + return self + + def __exit__(self, exc, value, tb): + if self.tmp: + self.tmp.cleanup() + + def root_dir(self) -> Path: + return Path(self.tmp.name) if self.tmp else self.directory + + def path(self, *paths: str) -> Path: + return Path(os.path.join(self.root_dir(), *paths)) + + def mkdir(self, *paths: str, is_file=False) -> Path: + """ + Create a directory using the given path rooted in the working directory. + If a file is provided, the directory up to that file will be created instead. + Returns the full path. + """ + p = self.path(*paths) + + # Let's create the parent directories exist + p2 = pathlib.Path(p) + if is_file: + p2 = p2.parent + + p2.mkdir(parents=True, exist_ok=True) + + return p + + def print_tree(self): + + def tree(dir_path: pathlib.Path, prefix: str = ""): + """A recursive generator, given a directory Path object + will yield a visual tree structure line by line + with each line prefixed by the same characters + """ + # prefix components: + space = " " + branch = "│ " + # pointers: + tee = "├── " + last = "└── " + + contents = list(dir_path.iterdir()) + # contents each get pointers that are ├── with a final └── : + pointers = [tee] * (len(contents) - 1) + [last] + for pointer, path in zip(pointers, contents): + is_dir = path.is_dir() + yield prefix + pointer + path.name + ("/" if is_dir else "") + if path.is_dir(): # extend the prefix and recurse: + extension = branch if pointer == tee else space + # i.e. space because last, └── , above so no more | + yield from tree(path, prefix=prefix + extension) + + root = self.root_dir() + print(root) + for line in tree(pathlib.Path(root)): + print(line) + + +class CAHierarchy: + """ + Utilities to manage a CA hierarchy on disk. + """ + + def __init__(self, base: Path, name: str) -> None: + self.name = name + self.directory = base + + self.root_cert_path = self.directory / "root.pem" + self.root_key_path = self.directory / "root.key" + + if self.root_key_path.exists() and self.root_cert_path.exists(): + # Root certificate already exists, re-using them + self.root_cert = x509.load_pem_x509_certificate( + self.root_cert_path.read_bytes() + ) + self.root_key = serialization.load_pem_private_key( + self.root_key_path.read_bytes(), None + ) + + logging.info(f"CA Hierarchy loaded from disk: {self.name}") + elif self.root_key_path.exists() or self.root_cert_path.exists(): + # Only one of the two file exists, bailing out + logging.error("root certificate partially exist on disk, stopping early") + sys.exit(1) + else: + # Start generating the root certificate + self.root_key = rsa.generate_private_key( + public_exponent=65537, key_size=2048 + ) + rand_suffix = "".join( + random.choices(string.ascii_letters + string.digits, k=16) + ) + root_cert_subject = x509.Name( + [ + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "CSA"), + x509.NameAttribute( + NameOID.COMMON_NAME, "TC_PAVS root " + rand_suffix + ), + ] + ) + self.root_cert = ( + x509.CertificateBuilder() + .subject_name(root_cert_subject) + .issuer_name(root_cert_subject) + .public_key(self.root_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.now(datetime.timezone.utc)) + .not_valid_after( + datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(days=1) + ) + .add_extension( + x509.BasicConstraints(ca=True, path_length=None), critical=True + ) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=True, + crl_sign=True, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .add_extension( + x509.SubjectKeyIdentifier.from_public_key( + self.root_key.public_key() + ), + critical=False, + ) + .sign(self.root_key, hashes.SHA256()) + ) + + self._save_cert("root", self.root_cert, self.root_key, False) + + logging.info(f"CA Hierarchy generated: {self.name}") + + def _save_cert( + self, + name: str, + cert: x509.Certificate, + key: Union[CERTIFICATE_PRIVATE_KEY_TYPES, None], + bundle_root: bool, + ) -> tuple[str, str]: + """ + Private method that help with saving certificate and key to the hierarchy folder. + This tool isn't meant to be used in production, but instead to help with development + and as such have the goal to make the CA hierarchy as available as possible, which in + turn make it very unsecure. + """ + cert_path = self.directory / f"{name}.pem" + key_path = self.directory / f"{name}.key" + + if key: + with open(key_path, "wb") as f: + f.write( + key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + + with open(cert_path, "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + + if bundle_root: + f.write(b"\n") + f.write(self.root_cert.public_bytes(serialization.Encoding.PEM)) + + return (key_path, cert_path) + + def _sign_cert( + self, dns: str, public_key: CERTIFICATE_PUBLIC_KEY_TYPES + ) -> x509.Certificate: + """ + Generate and sign a certificate. + """ + # Sign certificate + subject = x509.Name( + [ + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "CSA"), + x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, "TC_PAVS"), + x509.NameAttribute(NameOID.COMMON_NAME, dns), + ] + ) + + return ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(self.root_cert.subject) + .public_key(public_key) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.now(datetime.timezone.utc)) + .not_valid_after( + datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(hours=1) + ) + .add_extension( + x509.SubjectAlternativeName( + [ + # Describe what sites we want this certificate for. + # TODO Is it needed when we already have it in the CN field ? + x509.DNSName(dns), + ] + ), + critical=False, + ) + .add_extension( + x509.BasicConstraints(ca=False, path_length=None), + critical=True, + ) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=True, + data_encipherment=False, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .add_extension( + x509.ExtendedKeyUsage( + [ + x509.ExtendedKeyUsageOID.CLIENT_AUTH, + x509.ExtendedKeyUsageOID.SERVER_AUTH, + ] + ), + critical=False, + ) + .add_extension( + x509.SubjectKeyIdentifier.from_public_key(public_key), + critical=False, + ) + .add_extension( + x509.AuthorityKeyIdentifier.from_issuer_subject_key_identifier( + self.root_cert.extensions.get_extension_for_class( + x509.SubjectKeyIdentifier + ).value + ), + critical=False, + ) + .sign(self.root_key, hashes.SHA256()) + ) + + def gen_cert(self, dns: str, csr: str, override=False) -> tuple[Path, Path, bool]: + """ + Generate a certificate signed by this CA hierarchy using the provided CSR. + Returns the path to the key, cert, and whether it was reused or not. + """ + signing_request = x509.load_pem_x509_csr(csr) + signing_request.public_key() + + # If we don't always override, first check if an existing keypair already exists + if not override: + cert_path = self.directory / f"{dns}.pem" + key_path = self.directory / f"{dns}.key" + + if cert_path.exists() and key_path.exists(): + return (key_path, cert_path, True) + + # Sign certificate + cert = self._sign_cert(dns, csr.public_key()) + + # Save that information to disk + (key_path, cert_bundle_path) = self._save_cert( + dns, cert, None, bundle_root=True + ) + + logging.debug("leaf generated. dns=%s; path=%s", dns, cert_bundle_path) + + return (key_path, cert_bundle_path, False) + + def gen_keypair(self, dns: str, override=False) -> tuple[Path, Path, bool]: + """ + Generate a private key as well as the associated certificate signed by this CA + hierarchy. Returns the path to the key, cert, and whether it was reused or not. + """ + + # If we don't always override, first check if an existing keypair already exists + if not override: + cert_path = self.directory / f"{dns}.pem" + key_path = self.directory / f"{dns}.key" + + if cert_path.exists() and key_path.exists(): + return (key_path, cert_path, True) + + # Generate private key + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + + # Sign certificate + cert = self._sign_cert(dns, key.public_key()) + + # Save that information to disk + (key_path, cert_bundle_path) = self._save_cert(dns, cert, key, bundle_root=True) + + logging.debug("leaf generated. dns=%s; path=%s", dns, cert_bundle_path) + + return (key_path, cert_bundle_path, False) + + +app = FastAPI() +# Couldn't find how to do DI with state created in __main__ without using +# global variables, so using those. +wd: WorkingDirectory = None +device_hierarchy: CAHierarchy = None + + +@app.get("/", response_class=HTMLResponse) +def root(): + # TODO Include the UI here + + with open("index.html", "r") as f: + return f.read() + + +@app.post("/streams", status_code=201) +def create_stream(): + # Find the last registered stream + dirs = [d for d in pathlib.Path(wd.path("streams")).iterdir() if d.is_dir()] + last_stream = int(dirs[-1].name) if len(dirs) > 0 else 0 + print(f"last_stream={last_stream}") + stream_id = last_stream + 1 + + wd.mkdir("streams", str(stream_id)) + + return {"stream_id": stream_id} + + +@app.get("/streams") +def list_streams(): + dirs = [d for d in pathlib.Path(wd.path("streams")).iterdir() if d.is_dir()] + + # TODO Files can be multiple directory deep, need to reconstruct the tree here + streams = [{"id": d.name, "files": [f.name for f in d.iterdir()]} for d in dirs] + + return {"streams": streams} + + +# TODO app.put("/streams/{stream_id}") +# To conform to the ingest spec, support of that path with discarding the body +# and not doing anything. + +@app.put("/streams/{stream_id}/{file_path:path}", status_code=202) +async def segment_upload(file_path: str, stream_id: int, req: Request): + """Extract the parsed version of a client certificate. + See https://docs.python.org/3/library/ssl.html#ssl.SSLSocket.getpeercert + for the exact content. + """ + cert_details = req.scope["transport"].get_extra_info("ssl_object").getpeercert() + + logging.debug(f"segment_upload. stream_id={stream_id} file_path:{file_path}") + + if not wd.path("streams", str(stream_id)).exists(): + raise HTTPException(404, detail="Stream doesn't exists") + + dst = wd.mkdir("streams", str(stream_id), file_path, is_file=True) + + with open(dst.with_suffix(dst.suffix + ".crt"), "w") as f: + f.write(json.dumps(cert_details)) + + with open(dst, "wb") as f: + async for chunk in req.stream(): + f.write(chunk) + + return Response(status_code=202) + + +@app.get("/streams/probe/{stream_id}/{file_path:path}") +def ffprobe_check(file_path: str, stream_id: int): + + p = wd.path("streams", str(stream_id), file_path) + + if not p.exists(): + return HTTPException(404, detail="Stream doesn't exists") + + proc = subprocess.run( + ["ffprobe", "-show_streams", "-show_format", "-output_format", "json", str(p.absolute())], + capture_output=True + ) + + if proc.returncode != 0: + # TODO Add more details (maybe stderr) to the response + return HTTPException(500) + + return json.loads(proc.stdout) + + +# TODO app.post("streams/convert/{stream_id}") +# Will execute a ffmpeg conversion of the uploaded stream to a regular mp4 +# for easier consumption. Need to double check first if Firefox/Chrome can +# read DASH media with CMAF tracks. If they can no need to actually convert +# to a new format. + + +@app.get("/certs", status_code=200) +def list_certs(): + server = [f.name for f in pathlib.Path(wd.path("server")).iterdir()] + device = [f.name for f in pathlib.Path(wd.path("device")).iterdir()] + + return {"server": server, "device": device} + + +@app.post("/certs/{name}/keypair") +def create_client_keypair(name: str, override: bool = True): + (key, cert, created) = device_hierarchy.gen_keypair(name, override) + + return {key, cert, created} + + +class SignClientCertificate(BaseModel): + csr: str + + +@app.post("/certs/{name}/issue") +def sign_client_certificate( + name: str, req: SignClientCertificate, override: bool = True +): + (key, cert, created) = device_hierarchy.gen_cert(name, req.csr, override) + + return {key, cert, created} + + +def run(host: Optional[str], port: Optional[int], working_directory: Optional[str], dns: Optional[str]): + # For the Test Harness implementation, will need to find a way to return a reference + # to the server running in the background. This function currently doesn't return. + with WorkingDirectory(working_directory) as directory: + + # Sharing state with the various endpoints + wd = directory + + dns = "localhost" if dns is None else f"{dns}._http._tcp_.local." + + # Create CA hierarchies (for webserver and devices) + device_hierarchy = CAHierarchy(directory.mkdir("certs", "device"), "device") + server_hierarchy = CAHierarchy(directory.mkdir("certs", "server"), "server") + (server_key_file, server_cert_file, _) = server_hierarchy.gen_keypair(dns, override=True) + + # mDNS configuration. Registration only happen if the dns isn't localhost. + zeroconf = Zeroconf() + svc_info = None + + if dns != "localhost": + svc_info = ServiceInfo( + "_http._tcp.local.", + name=dns, + addresses=[socket.inet_aton("127.0.0.1")], + port=1234, + ) + # Advertise over mDNS + logging.info("Advertising the service as %s", svc_info) + zeroconf.register_service(svc_info) + + # Streams holder + wd.mkdir("streams") + + # Setup the web server + try: + uvicorn.run( + app, + host=host, + port=port, + ssl_keyfile=server_key_file, + ssl_certfile=server_cert_file, + ssl_cert_reqs=ssl.CERT_OPTIONAL, + ssl_ca_certs=device_hierarchy.root_cert_path, + ) + finally: + if dns != "localhost": + zeroconf.unregister_service(svc_info) + + directory.print_tree() + + +# TODO UI +# html page to provide a good way to see uploaded content (file "browser" + video player) + +if __name__ == "__main__": + logging.basicConfig( + format="%(asctime)s|%(name)-8s|%(levelname)-5s|%(message)s", + level=logging.DEBUG, + datefmt="%H:%M:%S", + ) + + parser = argparse.ArgumentParser( + prog="push_av_tool.py", + description="Tooling to help test Matter's Push AV capabilities", + ) + + parser.add_argument("--host", default="localhost") + parser.add_argument("--port", default=1234) + parser.add_argument( + "--working-directory", + help="Where to store content like certificates or uploaded streams. " + "Default to a temporary directory.", + ) + parser.add_argument( + "--dns", help="A mDNS record to adversise, or none if left empty." + ) + + args = parser.parse_args() + + run(args.host, args.port, args.working_directory, args.dns) From 2bc61708db416d6bcbd93cc3a33db75b8fba32ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Wed, 9 Oct 2024 16:24:42 -0700 Subject: [PATCH 02/27] Initial implemention of a push av server --- src/python_testing/TC_PAVS_1_0.py | 28 ---------------------------- src/tools/push_av_server/server.py | 5 +---- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/src/python_testing/TC_PAVS_1_0.py b/src/python_testing/TC_PAVS_1_0.py index 288eb3626e34a0..90d8d42157806e 100644 --- a/src/python_testing/TC_PAVS_1_0.py +++ b/src/python_testing/TC_PAVS_1_0.py @@ -1,33 +1,5 @@ -import logging -import time -import http.server -import ssl -import threading -import cryptography -import tempfile -import os.path -import datetime -from functools import partial -import shutil -# What's the stands of the Matter project on depending on requests ? -# TODO Actually do we really need requests once we have matter cameras in place? -import requests -import subprocess -import json -import socket -import sys - import push_av_server -import requests.adapters -from zeroconf import ServiceInfo, Zeroconf -from urllib.parse import urlparse - -from cryptography import x509 -from cryptography.x509.oid import NameOID -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import rsa - import chip.clusters as Clusters from chip.clusters import ClusterObjects as ClusterObjects from matter_testing_support import (ClusterAttributeChangeAccumulator, MatterBaseTest, TestStep, default_matter_test_main, diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index d61ce0461a3562..61daf4dc228469 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -1,13 +1,9 @@ -# TODO Move this one src/tools/ instead of having it here import logging import ssl from typing import Optional, Union import tempfile import os.path import datetime - -# What's the stands of the Matter project on depending on requests ? -# TODO Actually do we really need requests once we have matter cameras in place? import argparse import pathlib from pathlib import Path @@ -99,6 +95,7 @@ def mkdir(self, *paths: str, is_file=False) -> Path: return p def print_tree(self): + # TODO Convert this helper to build a HTML representation for use in the UI def tree(dir_path: pathlib.Path, prefix: str = ""): """A recursive generator, given a directory Path object From e0d8388ab94a26162f82c95f4466aad5371cf75d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 15 Oct 2024 17:11:43 -0700 Subject: [PATCH 03/27] restore global variable access --- src/tools/push_av_server/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index 61daf4dc228469..43cea3bad8f86a 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -501,6 +501,7 @@ def sign_client_certificate( def run(host: Optional[str], port: Optional[int], working_directory: Optional[str], dns: Optional[str]): + global wd, device_hierarchy # For the Test Harness implementation, will need to find a way to return a reference # to the server running in the background. This function currently doesn't return. with WorkingDirectory(working_directory) as directory: From 7dc4a5fca5fa561d440e6a3d5cab2205c54a33a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 15 Oct 2024 17:12:26 -0700 Subject: [PATCH 04/27] fix cryptography warnings --- src/tools/push_av_server/server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index 43cea3bad8f86a..c120bc25595f55 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -22,8 +22,8 @@ from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.asymmetric.types import ( - CERTIFICATE_PUBLIC_KEY_TYPES, - CERTIFICATE_PRIVATE_KEY_TYPES, + CertificatePublicKeyTypes, + CertificateIssuerPrivateKeyTypes, ) @@ -213,7 +213,7 @@ def _save_cert( self, name: str, cert: x509.Certificate, - key: Union[CERTIFICATE_PRIVATE_KEY_TYPES, None], + key: Union[CertificateIssuerPrivateKeyTypes, None], bundle_root: bool, ) -> tuple[str, str]: """ @@ -245,7 +245,7 @@ def _save_cert( return (key_path, cert_path) def _sign_cert( - self, dns: str, public_key: CERTIFICATE_PUBLIC_KEY_TYPES + self, dns: str, public_key: CertificatePublicKeyTypes ) -> x509.Certificate: """ Generate and sign a certificate. From 6f270f422850a09f3e21de0fc34ef99044320fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 15 Oct 2024 17:24:21 -0700 Subject: [PATCH 05/27] multiplex audio and video tracks --- .../push_av_server/generate_cmaf_content.sh | 154 +++++------------- 1 file changed, 45 insertions(+), 109 deletions(-) diff --git a/src/tools/push_av_server/generate_cmaf_content.sh b/src/tools/push_av_server/generate_cmaf_content.sh index 194f4029326a26..68cb75f22397ab 100755 --- a/src/tools/push_av_server/generate_cmaf_content.sh +++ b/src/tools/push_av_server/generate_cmaf_content.sh @@ -7,120 +7,56 @@ PUBLISHING_ENDPOINT=${PUBLISHING_ENDPOINT:-https://localhost:1234/stream/1} HTTP_OPTS=ca_file=~/.pavstest/certs/server/root.pem,cert_file=~/.pavstest/certs/device/dev.pem,key_file=~/.pavstest/certs/device/dev.key ffmpeg -hide_banner \ - -re -f lavfi -i " + -re -f lavfi -i " testsrc2=size=1280x720:rate=25, drawbox=x=0:y=0:w=700:h=50:c=black@.6:t=fill, drawtext=x= 5:y=5:fontsize=54:fontcolor=white:text='%{pts\:gmtime\:$(date +%s)\:%Y-%m-%d}', drawtext=x=345:y=5:fontsize=54:fontcolor=white:timecode='$(date -u '+%H\:%M\:%S')\:00':rate=25:tc24hmax=1, setparams=field_mode=prog:range=tv:color_primaries=bt709:color_trc=bt709:colorspace=bt709, format=yuv420p" \ - -re -f lavfi -i " + -re -f lavfi -i " sine=f=1000:r=48000:samples_per_frame='st(0,mod(n,5)); 1602-not(not(eq(ld(0),1)+eq(ld(0),3)))'" \ - -shortest \ - -fflags genpts \ - \ - -filter_complex " - [0:v]drawtext=x=(w-text_w)-5:y=5:fontsize=54:fontcolor=white:text='720p':box=1:boxcolor=black@.6:boxborderw=5[v720p]; - [0:v]drawtext=x=(w-text_w)-5:y=5:fontsize=54:fontcolor=white:text='360p':box=1:boxcolor=black@.6:boxborderw=5,scale=640x360[v360p] + -shortest \ + -fflags genpts \ + \ + -filter_complex " + [0:v]drawtext=x=(w-text_w)-5:y=5:fontsize=54:fontcolor=white:text='720p':box=1:boxcolor=black@.6:boxborderw=5 " \ - \ - -map [v720p] \ - -c:v libx264 \ - -preset:v veryfast \ - -tune zerolatency \ - -profile:v main \ - -crf:v 23 -bufsize:v:0 2250k -maxrate:v 2500k \ - -g:v 100000 -keyint_min:v 50000 -force_key_frames:v "expr:gte(t,n_forced*2)" \ - -x264opts no-open-gop=1 \ - -bf 2 -b_strategy 2 -refs 1 \ - -rc-lookahead 24 \ - -export_side_data prft \ - -field_order progressive -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv \ - -pix_fmt yuv420p \ - -f mp4 \ - -frag_duration "$((1 * 1000 * 1000))" \ - -min_frag_duration "$((1 * 1000 * 1000))" \ - -write_prft wallclock \ - -use_editlist 0 \ - -movflags "+cmaf+dash+delay_moov+skip_sidx+skip_trailer+frag_custom" \ - \ - -method PUT \ - -multiple_requests 1 \ - -chunked_post 1 \ - -send_expect_100 1 \ - -headers "DASH-IF-Ingest: 1.1" \ - -headers "Host: localhost:8080" \ - -content_type "" \ - -icy 0 \ - -rw_timeout "$((200 * 1000 * 1000))" \ - -reconnect 1 \ - -reconnect_at_eof 1 \ - -reconnect_on_network_error 1 \ - -reconnect_on_http_error 4xx,5xx \ - -reconnect_delay_max 2 \ - -http_opts $HTTP_OPTS \ - "$PUBLISHING_ENDPOINT/cmaf/example.str/Switching(video)/video-720p.cmfv" \ - \ - -map [v360p] \ - -c:v libx264 \ - -preset:v veryfast \ - -tune zerolatency \ - -profile:v main \ - -crf:v 23 -bufsize:v:0 2250k -maxrate:v 2500k \ - -g:v 100000 -keyint_min:v 50000 -force_key_frames:v "expr:gte(t,n_forced*2)" \ - -x264opts no-open-gop=1 \ - -bf 2 -b_strategy 2 -refs 1 \ - -rc-lookahead 24 \ - -export_side_data prft \ - -field_order progressive -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv \ - -pix_fmt yuv420p \ - -f mp4 \ - -frag_duration "$((1 * 1000 * 1000))" \ - -min_frag_duration "$((1 * 1000 * 1000))" \ - -write_prft wallclock \ - -use_editlist 0 \ - -movflags "+cmaf+dash+delay_moov+skip_sidx+skip_trailer+frag_custom" \ - \ - -method PUT \ - -multiple_requests 1 \ - -chunked_post 1 \ - -send_expect_100 1 \ - -headers "DASH-IF-Ingest: 1.1" \ - -headers "Host: localhost:8080" \ - -content_type "" \ - -icy 0 \ - -rw_timeout "$((200 * 1000 * 1000))" \ - -reconnect 1 \ - -reconnect_at_eof 1 \ - -reconnect_on_network_error 1 \ - -reconnect_on_http_error 4xx,5xx \ - -reconnect_delay_max 2 \ - -http_opts $HTTP_OPTS \ - "$PUBLISHING_ENDPOINT/cmaf/example.str/Switching(video)/video-360p.cmfv" \ - \ - -map 1:a \ - -c:a aac \ - -b:a 64k \ - -f mp4 \ - -frag_duration "$((1 * 1000 * 1000))" \ - -min_frag_duration "$((1 * 1000 * 1000))" \ - -write_prft wallclock \ - -use_editlist 0 \ - -movflags "+cmaf+dash+delay_moov+skip_sidx+skip_trailer+frag_custom" \ - \ - -method PUT \ - -multiple_requests 1 \ - -chunked_post 1 \ - -send_expect_100 1 \ - -headers "DASH-IF-Ingest: 1.1" \ - -headers "Host: localhost:8080" \ - -content_type "" \ - -icy 0 \ - -rw_timeout "$((200 * 1000 * 1000))" \ - -reconnect 1 \ - -reconnect_at_eof 1 \ - -reconnect_on_network_error 1 \ - -reconnect_on_http_error 4xx,5xx \ - -reconnect_delay_max 2 \ - -http_opts $HTTP_OPTS \ - "$PUBLISHING_ENDPOINT/cmaf/example.str/Switching(audio)/audio-64k.cmfa" + \ + -c:a aac \ + -b:a 64k \ + -c:v libx264 \ + -preset:v veryfast \ + -tune zerolatency \ + -profile:v main \ + -crf:v 23 -bufsize:v:0 2250k -maxrate:v 2500k \ + -g:v 100000 -keyint_min:v 50000 -force_key_frames:v "expr:gte(t,n_forced*2)" \ + -x264opts no-open-gop=1 \ + -bf 2 -b_strategy 2 -refs 1 \ + -rc-lookahead 24 \ + -export_side_data prft \ + -field_order progressive -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv \ + -pix_fmt yuv420p \ + -f mp4 \ + -frag_duration "$((1 * 1000 * 1000))" \ + -min_frag_duration "$((1 * 1000 * 1000))" \ + -write_prft wallclock \ + -use_editlist 0 \ + -movflags "+cmaf+dash+delay_moov+skip_sidx+skip_trailer+frag_custom" \ + \ + -method PUT \ + -multiple_requests 1 \ + -chunked_post 1 \ + -send_expect_100 1 \ + -headers "DASH-IF-Ingest: 1.1" \ + -headers "Host: localhost:8080" \ + -content_type "" \ + -icy 0 \ + -rw_timeout "$((200 * 1000 * 1000))" \ + -reconnect 1 \ + -reconnect_at_eof 1 \ + -reconnect_on_network_error 1 \ + -reconnect_on_http_error 4xx,5xx \ + -reconnect_delay_max 2 \ + -http_opts $HTTP_OPTS \ + "$PUBLISHING_ENDPOINT/cmaf/example/video-720p.cmfv" From f4ac5f862a915c3f90a091ab985719e0951f176d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 22 Oct 2024 14:23:06 -0700 Subject: [PATCH 06/27] Correct EKU based on hierarchy's kind --- src/tools/push_av_server/server.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index c120bc25595f55..42d0273f55fdf8 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -259,6 +259,9 @@ def _sign_cert( ] ) + extended_key_usage = [x509.ExtendedKeyUsageOID.CLIENT_AUTH] if self.name == "device" else [ + x509.ExtendedKeyUsageOID.SERVER_AUTH] + return ( x509.CertificateBuilder() .subject_name(subject) @@ -299,12 +302,7 @@ def _sign_cert( critical=True, ) .add_extension( - x509.ExtendedKeyUsage( - [ - x509.ExtendedKeyUsageOID.CLIENT_AUTH, - x509.ExtendedKeyUsageOID.SERVER_AUTH, - ] - ), + x509.ExtendedKeyUsage(extended_key_usage), critical=False, ) .add_extension( From 905edc7eced9cb681fe74d7e5a41e4f8b2dfba2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 22 Oct 2024 14:32:16 -0700 Subject: [PATCH 07/27] Fix listing stream files --- src/tools/push_av_server/server.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index 42d0273f55fdf8..90f79678e726a9 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -408,8 +408,10 @@ def create_stream(): def list_streams(): dirs = [d for d in pathlib.Path(wd.path("streams")).iterdir() if d.is_dir()] - # TODO Files can be multiple directory deep, need to reconstruct the tree here - streams = [{"id": d.name, "files": [f.name for f in d.iterdir()]} for d in dirs] + def stream_files(dir: Path): + return [f.relative_to(dir) for f in dir.glob("**/*") if f.is_file()] + + streams = [{"id": d.name, "files": stream_files(d)} for d in dirs] return {"streams": streams} From 1695ca6688329657490f54ed97096dbe7410c9dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 22 Oct 2024 15:50:05 -0700 Subject: [PATCH 08/27] Run push av server in the background --- src/python_testing/TC_PAVS_1_0.py | 13 +++++++++++++ src/tools/push_av_server/server.py | 12 ++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/python_testing/TC_PAVS_1_0.py b/src/python_testing/TC_PAVS_1_0.py index 90d8d42157806e..c740c17a4866be 100644 --- a/src/python_testing/TC_PAVS_1_0.py +++ b/src/python_testing/TC_PAVS_1_0.py @@ -1,5 +1,6 @@ import push_av_server +from multiprocessing import Process import chip.clusters as Clusters from chip.clusters import ClusterObjects as ClusterObjects from matter_testing_support import (ClusterAttributeChangeAccumulator, MatterBaseTest, TestStep, default_matter_test_main, @@ -14,6 +15,18 @@ class TC_PAVS_1_0(MatterBaseTest): for a better integration. It is not designed to be merged nor does it actually run. """ + def setup_class(self): + super().setup_class() + + self.proc = Process(target=push_av_server.run, + args=("127.0.0.1", 1234, None, "localhost"), + daemon=True) + self.proc.start() + + def teardown_class(self): + super().teardown_class() + self.proc.terminate() + def steps_TC_PAVS_1_0(self): return [TestStep(1, "Commissioning, already done", is_commissioning=True), TestStep(2, "Install CA onto the device"), diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index 90f79678e726a9..d4d47f0fcbe89b 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -385,8 +385,6 @@ def gen_keypair(self, dns: str, override=False) -> tuple[Path, Path, bool]: @app.get("/", response_class=HTMLResponse) def root(): - # TODO Include the UI here - with open("index.html", "r") as f: return f.read() @@ -474,8 +472,8 @@ def ffprobe_check(file_path: str, stream_id: int): @app.get("/certs", status_code=200) def list_certs(): - server = [f.name for f in pathlib.Path(wd.path("server")).iterdir()] - device = [f.name for f in pathlib.Path(wd.path("device")).iterdir()] + server = [f.name for f in pathlib.Path(wd.path("certs", "server")).iterdir()] + device = [f.name for f in pathlib.Path(wd.path("certs", "device")).iterdir()] return {"server": server, "device": device} @@ -502,8 +500,10 @@ def sign_client_certificate( def run(host: Optional[str], port: Optional[int], working_directory: Optional[str], dns: Optional[str]): global wd, device_hierarchy - # For the Test Harness implementation, will need to find a way to return a reference - # to the server running in the background. This function currently doesn't return. + """Run the reference server. This function will not return. + In the context where a background server is required, the multiprocessing.Process object + can be used. + """ with WorkingDirectory(working_directory) as directory: # Sharing state with the various endpoints From 1b57e9cc74a48a283eed938621d5f830810666e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Thu, 7 Nov 2024 13:09:17 -0800 Subject: [PATCH 09/27] change ca validity default + prep change to make it customizable --- src/tools/push_av_server/server.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index d4d47f0fcbe89b..abc697711cab6e 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -131,6 +131,8 @@ class CAHierarchy: Utilities to manage a CA hierarchy on disk. """ + default_ca_duration = datetime.timedelta(days=365.25*20) + def __init__(self, base: Path, name: str) -> None: self.name = name self.directory = base @@ -176,8 +178,7 @@ def __init__(self, base: Path, name: str) -> None: .serial_number(x509.random_serial_number()) .not_valid_before(datetime.datetime.now(datetime.timezone.utc)) .not_valid_after( - datetime.datetime.now(datetime.timezone.utc) - + datetime.timedelta(days=1) + datetime.datetime.now(datetime.timezone.utc) + self.default_ca_duration ) .add_extension( x509.BasicConstraints(ca=True, path_length=None), critical=True @@ -245,7 +246,10 @@ def _save_cert( return (key_path, cert_path) def _sign_cert( - self, dns: str, public_key: CertificatePublicKeyTypes + self, + dns: str, + public_key: CertificatePublicKeyTypes, + duration: datetime.timedelta ) -> x509.Certificate: """ Generate and sign a certificate. @@ -270,8 +274,7 @@ def _sign_cert( .serial_number(x509.random_serial_number()) .not_valid_before(datetime.datetime.now(datetime.timezone.utc)) .not_valid_after( - datetime.datetime.now(datetime.timezone.utc) - + datetime.timedelta(hours=1) + datetime.datetime.now(datetime.timezone.utc) + duration ) .add_extension( x509.SubjectAlternativeName( @@ -320,7 +323,7 @@ def _sign_cert( .sign(self.root_key, hashes.SHA256()) ) - def gen_cert(self, dns: str, csr: str, override=False) -> tuple[Path, Path, bool]: + def gen_cert(self, dns: str, csr: str, override=False, duration: datetime.timedelta = datetime.timedelta(hours=1)) -> tuple[Path, Path, bool]: """ Generate a certificate signed by this CA hierarchy using the provided CSR. Returns the path to the key, cert, and whether it was reused or not. @@ -337,7 +340,7 @@ def gen_cert(self, dns: str, csr: str, override=False) -> tuple[Path, Path, bool return (key_path, cert_path, True) # Sign certificate - cert = self._sign_cert(dns, csr.public_key()) + cert = self._sign_cert(dns, csr.public_key(), duration) # Save that information to disk (key_path, cert_bundle_path) = self._save_cert( @@ -348,7 +351,7 @@ def gen_cert(self, dns: str, csr: str, override=False) -> tuple[Path, Path, bool return (key_path, cert_bundle_path, False) - def gen_keypair(self, dns: str, override=False) -> tuple[Path, Path, bool]: + def gen_keypair(self, dns: str, override=False, duration: datetime.timedelta = datetime.timedelta(hours=1)) -> tuple[Path, Path, bool]: """ Generate a private key as well as the associated certificate signed by this CA hierarchy. Returns the path to the key, cert, and whether it was reused or not. @@ -366,7 +369,7 @@ def gen_keypair(self, dns: str, override=False) -> tuple[Path, Path, bool]: key = rsa.generate_private_key(public_exponent=65537, key_size=2048) # Sign certificate - cert = self._sign_cert(dns, key.public_key()) + cert = self._sign_cert(dns, key.public_key(), duration) # Save that information to disk (key_path, cert_bundle_path) = self._save_cert(dns, cert, key, bundle_root=True) From e6dc11a4782d2f72ed6e438c5ea360a0f255e946 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Thu, 7 Nov 2024 16:20:53 -0800 Subject: [PATCH 10/27] Attempt to provide a UI to browse certs/streams --- src/tools/push_av_server/index.html | 21 +++--- src/tools/push_av_server/requirements.txt | 3 +- src/tools/push_av_server/server.py | 71 ++++++++++++++++++- src/tools/push_av_server/static/styles.css | 0 .../templates/certificates_details.html | 53 ++++++++++++++ .../templates/certificates_list.html | 34 +++++++++ .../templates/streams_list.html | 28 ++++++++ 7 files changed, 198 insertions(+), 12 deletions(-) create mode 100644 src/tools/push_av_server/static/styles.css create mode 100644 src/tools/push_av_server/templates/certificates_details.html create mode 100644 src/tools/push_av_server/templates/certificates_list.html create mode 100644 src/tools/push_av_server/templates/streams_list.html diff --git a/src/tools/push_av_server/index.html b/src/tools/push_av_server/index.html index 1e670229700f08..1a588afcc0f7c9 100644 --- a/src/tools/push_av_server/index.html +++ b/src/tools/push_av_server/index.html @@ -1,6 +1,7 @@ - + + Push AV Ref Server - - + + +

AV Push Server

Streams

@@ -83,5 +85,6 @@

Server certificates

Device certificates

- + + \ No newline at end of file diff --git a/src/tools/push_av_server/requirements.txt b/src/tools/push_av_server/requirements.txt index 6269524a672778..0fa0df4a1e2305 100644 --- a/src/tools/push_av_server/requirements.txt +++ b/src/tools/push_av_server/requirements.txt @@ -1,4 +1,5 @@ zeroconf cryptography uvicorn -fastapi \ No newline at end of file +fastapi +jinja2 diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index abc697711cab6e..0dbac16b1990f7 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -29,6 +29,8 @@ from fastapi import FastAPI, Request, HTTPException, Response from fastapi.responses import HTMLResponse +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates from pydantic import BaseModel # Monkey patch uvicorn to make the underlying transport available to us. @@ -380,24 +382,52 @@ def gen_keypair(self, dns: str, override=False, duration: datetime.timedelta = d app = FastAPI() +app.mount("/static", StaticFiles(directory="static"), name="static") +templates = Jinja2Templates(directory="templates") # Couldn't find how to do DI with state created in __main__ without using # global variables, so using those. wd: WorkingDirectory = None device_hierarchy: CAHierarchy = None +# UI website + @app.get("/", response_class=HTMLResponse) def root(): with open("index.html", "r") as f: return f.read() +@app.get("/ui/streams", response_class=HTMLResponse) +def ui_streams(request: Request): + s = list_streams() + return templates.TemplateResponse( + request=request, name="streams_list.html", context={"streams": s["streams"]} + ) + + +@app.get("/ui/certificates", response_class=HTMLResponse) +def ui_certificates(request: Request): + return templates.TemplateResponse( + request=request, name="certificates_list.html", context={"certs": list_certs()} + ) + + +@app.get("/ui/certificates/{hierarchy}/{name}", response_class=HTMLResponse) +def ui_certificates(request: Request, hierarchy: str, name: str): + context = certificate_details(hierarchy, name) + context["certs"] = list_certs() + + return templates.TemplateResponse(request=request, name="certificates_details.html", context=context) + +# APIs + + @app.post("/streams", status_code=201) def create_stream(): # Find the last registered stream dirs = [d for d in pathlib.Path(wd.path("streams")).iterdir() if d.is_dir()] last_stream = int(dirs[-1].name) if len(dirs) > 0 else 0 - print(f"last_stream={last_stream}") stream_id = last_stream + 1 wd.mkdir("streams", str(stream_id)) @@ -447,7 +477,7 @@ async def segment_upload(file_path: str, stream_id: int, req: Request): @app.get("/streams/probe/{stream_id}/{file_path:path}") -def ffprobe_check(file_path: str, stream_id: int): +def ffprobe_check(stream_id: int, file_path: str): p = wd.path("streams", str(stream_id), file_path) @@ -481,6 +511,43 @@ def list_certs(): return {"server": server, "device": device} +@app.get("/certs/{hierarchy}/{name}", status_code=200) +def certificate_details(hierarchy: str, name: str): + data = pathlib.Path(wd.path("certs", hierarchy, name)).read_bytes() + type = "key" if name.endswith(".key") else "cert" + + key = None + cert = None + if type == "key": + key = serialization.load_pem_private_key(data, None) + key = { + "key_size": key.key_size, + "private_key": key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ), + "public_key": key.public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.PKCS1, + ), + } + else: + cert = x509.load_pem_x509_certificate(data) + cert = { + "public_cert": cert.public_bytes(serialization.Encoding.PEM), + "serial_number": hex(cert.serial_number), + "not_valid_before": cert.not_valid_before_utc, + "not_valid_after": cert.not_valid_after_utc, + # public_key? fingerprint? + "issuer": cert.issuer.rfc4514_string(), + "subject": cert.subject.rfc4514_string(), + "extensions": [str(ext) for ext in cert.extensions] + } + + return {"type": type, "key": key, "cert": cert} + + @app.post("/certs/{name}/keypair") def create_client_keypair(name: str, override: bool = True): (key, cert, created) = device_hierarchy.gen_keypair(name, override) diff --git a/src/tools/push_av_server/static/styles.css b/src/tools/push_av_server/static/styles.css new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/tools/push_av_server/templates/certificates_details.html b/src/tools/push_av_server/templates/certificates_details.html new file mode 100644 index 00000000000000..b3eb9517131c80 --- /dev/null +++ b/src/tools/push_av_server/templates/certificates_details.html @@ -0,0 +1,53 @@ + + + + + Push AV Server + + + + + +
+

Certificates

+

Server

+
    + {% for cert_name in certs['server'] %} +
  • {{cert_name}}
    • + {% endfor %} +
+

Clients

+
    + {% for cert_name in certs['device'] %} +
  • {{cert_name}}
    • + {% endfor %} +
+
+
+
Type: {{type}}
+ {% if type == "cert" %} +
Serial Number: {{cert['serial_number']}}
+
Not Valid Before: {{cert['not_valid_before']}}
+
Not Valid After: {{cert['not_valid_after']}}
+
Issuer: {{cert['issuer']}}
+
Subject: {{cert['subject']}}
+
Extensions
+ {% for ext in cert['extensions'] %} +
{{ext}}
+ {% endfor %} + Public cert: + 
{{cert['public_cert']}}
+ {% else %} +
Key size: {{key['key_size']}}
+ Public key: + 
{{key['public_key']}}
+ Private key: + 
{{key['private_key']}}
+ {% endif %} +
+ + + \ No newline at end of file diff --git a/src/tools/push_av_server/templates/certificates_list.html b/src/tools/push_av_server/templates/certificates_list.html new file mode 100644 index 00000000000000..6da5b3a14b34b2 --- /dev/null +++ b/src/tools/push_av_server/templates/certificates_list.html @@ -0,0 +1,34 @@ + + + + + Push AV Server + + + + + +
+

Certificates

+

Server

+
    + {% for cert_name in certs['server'] %} +
  • {{cert_name}}
    • + {% endfor %} +
+

Clients

+
    + {% for cert_name in certs['device'] %} +
  • {{cert_name}}
    • + {% endfor %} +
+
+
+ +
+ + + \ No newline at end of file diff --git a/src/tools/push_av_server/templates/streams_list.html b/src/tools/push_av_server/templates/streams_list.html new file mode 100644 index 00000000000000..f5dfed8121762f --- /dev/null +++ b/src/tools/push_av_server/templates/streams_list.html @@ -0,0 +1,28 @@ + + + + + Push AV Server + + + + + + + + + + + {% for stream in streams %} + {% for file in stream['files'] %} + + + + + {% endfor %} + {% endfor %} + +
Stream IDFile
{{stream['id']}}{{file}}
+ + + \ No newline at end of file From 1afd43e2e0f765b5c54cf489327ab3904d889809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Fri, 8 Nov 2024 13:30:04 -0800 Subject: [PATCH 11/27] fix certificate handling of the cmaf generation script --- src/tools/push_av_server/generate_cmaf_content.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tools/push_av_server/generate_cmaf_content.sh b/src/tools/push_av_server/generate_cmaf_content.sh index 68cb75f22397ab..9e707e58854e48 100755 --- a/src/tools/push_av_server/generate_cmaf_content.sh +++ b/src/tools/push_av_server/generate_cmaf_content.sh @@ -3,8 +3,8 @@ PUBLISHING_ENDPOINT=${PUBLISHING_ENDPOINT:-https://localhost:1234/stream/1} -# TODO Handle dynamic value for those three variables -HTTP_OPTS=ca_file=~/.pavstest/certs/server/root.pem,cert_file=~/.pavstest/certs/device/dev.pem,key_file=~/.pavstest/certs/device/dev.key +# TODO Handle dynamic value for the certificates +CERT_ROOT_DIR=~/.pavstest ffmpeg -hide_banner \ -re -f lavfi -i " @@ -58,5 +58,8 @@ ffmpeg -hide_banner \ -reconnect_on_network_error 1 \ -reconnect_on_http_error 4xx,5xx \ -reconnect_delay_max 2 \ - -http_opts $HTTP_OPTS \ + -ca_file $CERT_ROOT_DIR/certs/server/root.pem \ + -cert_file $CERT_ROOT_DIR/certs/device/dev.pem \ + -key_file $CERT_ROOT_DIR/certs/device/dev.key \ + -tls_verify 1 \ "$PUBLISHING_ENDPOINT/cmaf/example/video-720p.cmfv" From f499d936b2f201ba79e20e1af00b3f09ee5155f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Fri, 8 Nov 2024 13:30:30 -0800 Subject: [PATCH 12/27] poc UI for stream files --- src/tools/push_av_server/server.py | 35 ++++++++++++--- .../templates/streams_details.html | 43 +++++++++++++++++++ .../templates/streams_list.html | 33 ++++++++------ 3 files changed, 92 insertions(+), 19 deletions(-) create mode 100644 src/tools/push_av_server/templates/streams_details.html diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index 0dbac16b1990f7..e5e40074faa2d2 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -28,7 +28,7 @@ from fastapi import FastAPI, Request, HTTPException, Response -from fastapi.responses import HTMLResponse +from fastapi.responses import HTMLResponse, FileResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from pydantic import BaseModel @@ -399,22 +399,42 @@ def root(): @app.get("/ui/streams", response_class=HTMLResponse) -def ui_streams(request: Request): +def ui_streams_list(request: Request): s = list_streams() return templates.TemplateResponse( request=request, name="streams_list.html", context={"streams": s["streams"]} ) +@app.get("/ui/streams/{stream_id}/{file_path:path}") +def ui_streams_details(request: Request, stream_id: int, file_path: str): + context = {} + context['streams'] = list_streams()['streams'] + context['stream_id'] = stream_id + context['file_path'] = file_path + + if file_path.endswith('.crt'): + context['type'] = 'cert' + p = wd.path("streams", str(stream_id), file_path) + with open(p, "r") as f: + context['cert'] = json.load(f) + else: + context['type'] = 'media' + context['probe'] = ffprobe_check(stream_id, file_path) + context['pretty_probe'] = json.dumps(context['probe'], sort_keys=True, indent=4) + + return templates.TemplateResponse(request=request, name="streams_details.html", context=context) + + @app.get("/ui/certificates", response_class=HTMLResponse) -def ui_certificates(request: Request): +def ui_certificates_list(request: Request): return templates.TemplateResponse( request=request, name="certificates_list.html", context={"certs": list_certs()} ) @app.get("/ui/certificates/{hierarchy}/{name}", response_class=HTMLResponse) -def ui_certificates(request: Request, hierarchy: str, name: str): +def ui_certificates_details(request: Request, hierarchy: str, name: str): context = certificate_details(hierarchy, name) context["certs"] = list_certs() @@ -476,6 +496,11 @@ async def segment_upload(file_path: str, stream_id: int, req: Request): return Response(status_code=202) +@app.get("/streams/{stream_id}/{file_path:path}") +async def segment_download(file_path: str, stream_id: int): + return FileResponse(wd.path("streams", str(stream_id), file_path)) + + @app.get("/streams/probe/{stream_id}/{file_path:path}") def ffprobe_check(stream_id: int, file_path: str): @@ -619,7 +644,7 @@ def run(host: Optional[str], port: Optional[int], working_directory: Optional[st if dns != "localhost": zeroconf.unregister_service(svc_info) - directory.print_tree() + # directory.print_tree() # TODO UI diff --git a/src/tools/push_av_server/templates/streams_details.html b/src/tools/push_av_server/templates/streams_details.html new file mode 100644 index 00000000000000..9807c330c14a0a --- /dev/null +++ b/src/tools/push_av_server/templates/streams_details.html @@ -0,0 +1,43 @@ + + + + + Push AV Server + + + + + +
+

Streams

+ + +
    + {% for stream in streams %} +
  • Stream {{stream['id']}} +
      + {% for file in stream['files'] %} +
    • {{file}}
      • + {% endfor %} +
    +
    • + {% endfor %} +
+
+ {% if type == 'cert' %} + 
{{cert}}
+ {% else %} +
+ +
+
+ 
{{pretty_probe}}
+
+ {% endif %} +
+ + + \ No newline at end of file diff --git a/src/tools/push_av_server/templates/streams_list.html b/src/tools/push_av_server/templates/streams_list.html index f5dfed8121762f..b4c1b359fe1a56 100644 --- a/src/tools/push_av_server/templates/streams_list.html +++ b/src/tools/push_av_server/templates/streams_list.html @@ -7,22 +7,27 @@ - - - - - + +
+

Streams

- {% for stream in streams %} - {% for file in stream['files'] %} -
- - - - {% endfor %} - {% endfor %} -
Stream IDFile
{{stream['id']}}{{file}}
+
    + {% for stream in streams %} +
  • Stream {{stream['id']}} +
      + {% for file in stream['files'] %} +
    • {{file}}
      • + {% endfor %} +
    +
    • + {% endfor %} +
+
+
\ No newline at end of file From e1182316569b1997ab2a11bc4486a81fffbaeecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Fri, 8 Nov 2024 13:34:59 -0800 Subject: [PATCH 13/27] lint and restyled --- src/python_testing/TC_PAVS_1_0.py | 9 ++------- src/tools/push_av_server/generate_cmaf_content.sh | 6 +++--- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/python_testing/TC_PAVS_1_0.py b/src/python_testing/TC_PAVS_1_0.py index c740c17a4866be..551d690afc8d79 100644 --- a/src/python_testing/TC_PAVS_1_0.py +++ b/src/python_testing/TC_PAVS_1_0.py @@ -1,12 +1,7 @@ import push_av_server from multiprocessing import Process -import chip.clusters as Clusters -from chip.clusters import ClusterObjects as ClusterObjects -from matter_testing_support import (ClusterAttributeChangeAccumulator, MatterBaseTest, TestStep, default_matter_test_main, - async_test_body) -from mobly import asserts -from test_plan_support import commission_if_required, if_feature_supported, read_attribute, verify_success +from matter_testing_support import (MatterBaseTest, TestStep, default_matter_test_main, async_test_body) class TC_PAVS_1_0(MatterBaseTest): @@ -58,7 +53,7 @@ async def test_TC_PAVS_1_0(self): self.step("3b") # Generate nonce # send TLSCertificateManagement.TLSClientCSR, receive TLSClientCSRResponse - push_av_server.device_hierarchy.gen_cert(name, csr) + push_av_server.device_hierarchy.gen_cert("device name", "csr") # send ProvisionClientCertificate, receive ProvisionClientCertificateResponse self.step(4) diff --git a/src/tools/push_av_server/generate_cmaf_content.sh b/src/tools/push_av_server/generate_cmaf_content.sh index 9e707e58854e48..6856c993539799 100755 --- a/src/tools/push_av_server/generate_cmaf_content.sh +++ b/src/tools/push_av_server/generate_cmaf_content.sh @@ -58,8 +58,8 @@ ffmpeg -hide_banner \ -reconnect_on_network_error 1 \ -reconnect_on_http_error 4xx,5xx \ -reconnect_delay_max 2 \ - -ca_file $CERT_ROOT_DIR/certs/server/root.pem \ - -cert_file $CERT_ROOT_DIR/certs/device/dev.pem \ - -key_file $CERT_ROOT_DIR/certs/device/dev.key \ + -ca_file "$CERT_ROOT_DIR/certs/server/root.pem" \ + -cert_file "$CERT_ROOT_DIR/certs/device/dev.pem" \ + -key_file "$CERT_ROOT_DIR/certs/device/dev.key" \ -tls_verify 1 \ "$PUBLISHING_ENDPOINT/cmaf/example/video-720p.cmfv" From 6b3521ebfec1cdcfabcd753cbc988a28ee0bb644 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 8 Nov 2024 21:39:41 +0000 Subject: [PATCH 14/27] Restyled by shfmt --- .../push_av_server/generate_cmaf_content.sh | 94 +++++++++---------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/src/tools/push_av_server/generate_cmaf_content.sh b/src/tools/push_av_server/generate_cmaf_content.sh index 6856c993539799..88dece7ac75d11 100755 --- a/src/tools/push_av_server/generate_cmaf_content.sh +++ b/src/tools/push_av_server/generate_cmaf_content.sh @@ -7,59 +7,59 @@ PUBLISHING_ENDPOINT=${PUBLISHING_ENDPOINT:-https://localhost:1234/stream/1} CERT_ROOT_DIR=~/.pavstest ffmpeg -hide_banner \ - -re -f lavfi -i " + -re -f lavfi -i " testsrc2=size=1280x720:rate=25, drawbox=x=0:y=0:w=700:h=50:c=black@.6:t=fill, drawtext=x= 5:y=5:fontsize=54:fontcolor=white:text='%{pts\:gmtime\:$(date +%s)\:%Y-%m-%d}', drawtext=x=345:y=5:fontsize=54:fontcolor=white:timecode='$(date -u '+%H\:%M\:%S')\:00':rate=25:tc24hmax=1, setparams=field_mode=prog:range=tv:color_primaries=bt709:color_trc=bt709:colorspace=bt709, format=yuv420p" \ - -re -f lavfi -i " + -re -f lavfi -i " sine=f=1000:r=48000:samples_per_frame='st(0,mod(n,5)); 1602-not(not(eq(ld(0),1)+eq(ld(0),3)))'" \ - -shortest \ - -fflags genpts \ - \ - -filter_complex " + -shortest \ + -fflags genpts \ + \ + -filter_complex " [0:v]drawtext=x=(w-text_w)-5:y=5:fontsize=54:fontcolor=white:text='720p':box=1:boxcolor=black@.6:boxborderw=5 " \ - \ - -c:a aac \ - -b:a 64k \ - -c:v libx264 \ - -preset:v veryfast \ - -tune zerolatency \ - -profile:v main \ - -crf:v 23 -bufsize:v:0 2250k -maxrate:v 2500k \ - -g:v 100000 -keyint_min:v 50000 -force_key_frames:v "expr:gte(t,n_forced*2)" \ - -x264opts no-open-gop=1 \ - -bf 2 -b_strategy 2 -refs 1 \ - -rc-lookahead 24 \ - -export_side_data prft \ - -field_order progressive -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv \ - -pix_fmt yuv420p \ - -f mp4 \ - -frag_duration "$((1 * 1000 * 1000))" \ - -min_frag_duration "$((1 * 1000 * 1000))" \ - -write_prft wallclock \ - -use_editlist 0 \ - -movflags "+cmaf+dash+delay_moov+skip_sidx+skip_trailer+frag_custom" \ - \ - -method PUT \ - -multiple_requests 1 \ - -chunked_post 1 \ - -send_expect_100 1 \ - -headers "DASH-IF-Ingest: 1.1" \ - -headers "Host: localhost:8080" \ - -content_type "" \ - -icy 0 \ - -rw_timeout "$((200 * 1000 * 1000))" \ - -reconnect 1 \ - -reconnect_at_eof 1 \ - -reconnect_on_network_error 1 \ - -reconnect_on_http_error 4xx,5xx \ - -reconnect_delay_max 2 \ - -ca_file "$CERT_ROOT_DIR/certs/server/root.pem" \ - -cert_file "$CERT_ROOT_DIR/certs/device/dev.pem" \ - -key_file "$CERT_ROOT_DIR/certs/device/dev.key" \ - -tls_verify 1 \ - "$PUBLISHING_ENDPOINT/cmaf/example/video-720p.cmfv" + \ + -c:a aac \ + -b:a 64k \ + -c:v libx264 \ + -preset:v veryfast \ + -tune zerolatency \ + -profile:v main \ + -crf:v 23 -bufsize:v:0 2250k -maxrate:v 2500k \ + -g:v 100000 -keyint_min:v 50000 -force_key_frames:v "expr:gte(t,n_forced*2)" \ + -x264opts no-open-gop=1 \ + -bf 2 -b_strategy 2 -refs 1 \ + -rc-lookahead 24 \ + -export_side_data prft \ + -field_order progressive -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv \ + -pix_fmt yuv420p \ + -f mp4 \ + -frag_duration "$((1 * 1000 * 1000))" \ + -min_frag_duration "$((1 * 1000 * 1000))" \ + -write_prft wallclock \ + -use_editlist 0 \ + -movflags "+cmaf+dash+delay_moov+skip_sidx+skip_trailer+frag_custom" \ + \ + -method PUT \ + -multiple_requests 1 \ + -chunked_post 1 \ + -send_expect_100 1 \ + -headers "DASH-IF-Ingest: 1.1" \ + -headers "Host: localhost:8080" \ + -content_type "" \ + -icy 0 \ + -rw_timeout "$((200 * 1000 * 1000))" \ + -reconnect 1 \ + -reconnect_at_eof 1 \ + -reconnect_on_network_error 1 \ + -reconnect_on_http_error 4xx,5xx \ + -reconnect_delay_max 2 \ + -ca_file "$CERT_ROOT_DIR/certs/server/root.pem" \ + -cert_file "$CERT_ROOT_DIR/certs/device/dev.pem" \ + -key_file "$CERT_ROOT_DIR/certs/device/dev.key" \ + -tls_verify 1 \ + "$PUBLISHING_ENDPOINT/cmaf/example/video-720p.cmfv" From ae8f652c5d4a2271f3df61e86325a85b931f8180 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 8 Nov 2024 21:39:47 +0000 Subject: [PATCH 15/27] Restyled by isort --- src/python_testing/TC_PAVS_1_0.py | 6 ++--- src/tools/push_av_server/server.py | 35 ++++++++++++------------------ 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/python_testing/TC_PAVS_1_0.py b/src/python_testing/TC_PAVS_1_0.py index 551d690afc8d79..a7d5bfca58142c 100644 --- a/src/python_testing/TC_PAVS_1_0.py +++ b/src/python_testing/TC_PAVS_1_0.py @@ -1,7 +1,7 @@ -import push_av_server - from multiprocessing import Process -from matter_testing_support import (MatterBaseTest, TestStep, default_matter_test_main, async_test_body) + +import push_av_server +from matter_testing_support import MatterBaseTest, TestStep, async_test_body, default_matter_test_main class TC_PAVS_1_0(MatterBaseTest): diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index e5e40074faa2d2..3a1b0bdc55dce2 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -1,41 +1,34 @@ +import argparse +import datetime +import json import logging -import ssl -from typing import Optional, Union -import tempfile import os.path -import datetime -import argparse import pathlib -from pathlib import Path import random -import string -import json import socket -import sys +import ssl +import string import subprocess - -from zeroconf import ServiceInfo, Zeroconf +import sys +import tempfile +from pathlib import Path +from typing import Optional, Union import uvicorn from cryptography import x509 -from cryptography.x509.oid import NameOID from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.hazmat.primitives.asymmetric.types import ( - CertificatePublicKeyTypes, - CertificateIssuerPrivateKeyTypes, -) - - -from fastapi import FastAPI, Request, HTTPException, Response -from fastapi.responses import HTMLResponse, FileResponse +from cryptography.hazmat.primitives.asymmetric.types import CertificateIssuerPrivateKeyTypes, CertificatePublicKeyTypes +from cryptography.x509.oid import NameOID +from fastapi import FastAPI, HTTPException, Request, Response +from fastapi.responses import FileResponse, HTMLResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from pydantic import BaseModel - # Monkey patch uvicorn to make the underlying transport available to us. # That will let us access the ssl context and get the client certificate information. from uvicorn.protocols.http.h11_impl import H11Protocol +from zeroconf import ServiceInfo, Zeroconf http_tools_protocol_old__should_upgrade = H11Protocol._should_upgrade From f4d338cb54b0aa576178574d19605a1e0cacd836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 26 Nov 2024 16:28:04 -0800 Subject: [PATCH 16/27] fix API ordering so that getting a file details work --- src/tools/push_av_server/server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index 3a1b0bdc55dce2..589375f6a25dc9 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -489,11 +489,6 @@ async def segment_upload(file_path: str, stream_id: int, req: Request): return Response(status_code=202) -@app.get("/streams/{stream_id}/{file_path:path}") -async def segment_download(file_path: str, stream_id: int): - return FileResponse(wd.path("streams", str(stream_id), file_path)) - - @app.get("/streams/probe/{stream_id}/{file_path:path}") def ffprobe_check(stream_id: int, file_path: str): @@ -514,6 +509,11 @@ def ffprobe_check(stream_id: int, file_path: str): return json.loads(proc.stdout) +@app.get("/streams/{stream_id}/{file_path:path}") +async def segment_download(file_path: str, stream_id: int): + return FileResponse(wd.path("streams", str(stream_id), file_path)) + + # TODO app.post("streams/convert/{stream_id}") # Will execute a ffmpeg conversion of the uploaded stream to a regular mp4 # for easier consumption. Need to double check first if Firefox/Chrome can From 01c5914983e65b4315d0eebf743440e8252ac7c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 26 Nov 2024 16:28:12 -0800 Subject: [PATCH 17/27] fix readme example --- src/tools/push_av_server/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/push_av_server/README.md b/src/tools/push_av_server/README.md index e717201138b645..9e2c4927e1f4ee 100644 --- a/src/tools/push_av_server/README.md +++ b/src/tools/push_av_server/README.md @@ -35,5 +35,5 @@ $ curl -XGET --cacert ~/.pavstest/certs/server/root.pem https://localhost:1234/s # Get detailed information about the uploaded media file. # This correspond to the ffprobe tool output -$ curl --cacert ~/.pavstest/certs/server/root.pem -XGET 'https://localhost:1234/streams/probe/1/cmaf/example.str/Switching(video)/video-720p.cmfv' +$ curl --cacert ~/.pavstest/certs/server/root.pem -XGET 'https://localhost:1234/streams/probe/1/cmaf/example/video-720p.cmfv' ``` From 8384867f3b495a152fbece95f751929e5ca3666a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 10 Dec 2024 16:39:53 -0800 Subject: [PATCH 18/27] Address code review feedback around certificate management --- src/tools/push_av_server/server.py | 105 +++++++++++++++++------------ 1 file changed, 61 insertions(+), 44 deletions(-) diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index 589375f6a25dc9..0561e63b559e1a 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -12,7 +12,7 @@ import sys import tempfile from pathlib import Path -from typing import Optional, Union +from typing import Optional, Union, Literal import uvicorn from cryptography import x509 @@ -128,8 +128,32 @@ class CAHierarchy: default_ca_duration = datetime.timedelta(days=365.25*20) - def __init__(self, base: Path, name: str) -> None: + client_key_usage_cert = x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=True, + data_encipherment=False, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False, + ) + server_key_usage_cert = x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False, + ) + + def __init__(self, base: Path, name: str, kind: Literal['server', 'client']) -> None: self.name = name + self.kind = kind self.directory = base self.root_cert_path = self.directory / "root.pem" @@ -176,7 +200,8 @@ def __init__(self, base: Path, name: str) -> None: datetime.datetime.now(datetime.timezone.utc) + self.default_ca_duration ) .add_extension( - x509.BasicConstraints(ca=True, path_length=None), critical=True + # We make it so that our root can only issue leaf certificates, no intermediate here. + x509.BasicConstraints(ca=True, path_length=0), critical=True ) .add_extension( x509.KeyUsage( @@ -258,55 +283,34 @@ def _sign_cert( ] ) - extended_key_usage = [x509.ExtendedKeyUsageOID.CLIENT_AUTH] if self.name == "device" else [ + extended_key_usage = [x509.ExtendedKeyUsageOID.CLIENT_AUTH] if self.kind == "client" else [ x509.ExtendedKeyUsageOID.SERVER_AUTH] - return ( - x509.CertificateBuilder() - .subject_name(subject) - .issuer_name(self.root_cert.subject) - .public_key(public_key) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.datetime.now(datetime.timezone.utc)) - .not_valid_after( - datetime.datetime.now(datetime.timezone.utc) + duration - ) - .add_extension( - x509.SubjectAlternativeName( - [ - # Describe what sites we want this certificate for. - # TODO Is it needed when we already have it in the CN field ? - x509.DNSName(dns), - ] - ), - critical=False, - ) + builder = (x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(self.root_cert.subject) + .public_key(public_key) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.now(datetime.timezone.utc)) + .not_valid_after( + datetime.datetime.now(datetime.timezone.utc) + duration + ) .add_extension( x509.BasicConstraints(ca=False, path_length=None), - critical=True, - ) + critical=False, + ) .add_extension( - x509.KeyUsage( - digital_signature=True, - content_commitment=False, - key_encipherment=True, - data_encipherment=False, - key_agreement=False, - key_cert_sign=False, - crl_sign=False, - encipher_only=False, - decipher_only=False, - ), + self.client_key_usage_cert if self.kind == "client" else self.server_key_usage_cert, critical=True, - ) + ) .add_extension( x509.ExtendedKeyUsage(extended_key_usage), critical=False, - ) + ) .add_extension( x509.SubjectKeyIdentifier.from_public_key(public_key), critical=False, - ) + ) .add_extension( x509.AuthorityKeyIdentifier.from_issuer_subject_key_identifier( self.root_cert.extensions.get_extension_for_class( @@ -314,9 +318,22 @@ def _sign_cert( ).value ), critical=False, - ) - .sign(self.root_key, hashes.SHA256()) ) + .add_extension(x509.CRLDistributionPoints([x509.DistributionPoint( + full_name=[x509.UniformResourceIdentifier("http://not.a.valid.website.com/some/path/to/a.crl")], + relative_name=None, + reasons=None, + crl_issuer=None + )]), critical=False) + ) + + if self.kind == 'server': + builder.add_extension( + x509.SubjectAlternativeName([x509.DNSName(dns)]), + critical=False, + ) + + return builder.sign(self.root_key, hashes.SHA256()) def gen_cert(self, dns: str, csr: str, override=False, duration: datetime.timedelta = datetime.timedelta(hours=1)) -> tuple[Path, Path, bool]: """ @@ -600,8 +617,8 @@ def run(host: Optional[str], port: Optional[int], working_directory: Optional[st dns = "localhost" if dns is None else f"{dns}._http._tcp_.local." # Create CA hierarchies (for webserver and devices) - device_hierarchy = CAHierarchy(directory.mkdir("certs", "device"), "device") - server_hierarchy = CAHierarchy(directory.mkdir("certs", "server"), "server") + device_hierarchy = CAHierarchy(directory.mkdir("certs", "device"), "device", "client") + server_hierarchy = CAHierarchy(directory.mkdir("certs", "server"), "server", "server") (server_key_file, server_cert_file, _) = server_hierarchy.gen_keypair(dns, override=True) # mDNS configuration. Registration only happen if the dns isn't localhost. From fd7b8e2a7df2e76185b3d4e4310ce8afd6b0ab62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 10 Dec 2024 16:44:40 -0800 Subject: [PATCH 19/27] add short example of using adhoc cert hierarchy --- src/python_testing/TC_PAVS_1_0.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/python_testing/TC_PAVS_1_0.py b/src/python_testing/TC_PAVS_1_0.py index a7d5bfca58142c..e5b3610d410610 100644 --- a/src/python_testing/TC_PAVS_1_0.py +++ b/src/python_testing/TC_PAVS_1_0.py @@ -133,6 +133,19 @@ async def test_TC_PAVS_1_0(self): srv.stop() + async def test_wrong_cert_chain(self): + """Intended as an example of how to install an unrecognised """ + srv = push_av_server.start("localhost", 1234) + srv.run_in_thread() + + wrong_chain = push_av_server.CAHierarchy(push_av_server.wd.mkdir("certs", "wrong"), "wrong chain", "client") + + wrong_chain.root_cert_path # Install onto the device + ["path_to_key", "path_to_cert"] = wrong_chain.gen_cert("dns", "csr") # Sign for the device + # Install onto the device + + # After setting up the device, trigger an upload which will use the wrong cert and should fail + if __name__ == "__main__": default_matter_test_main() From f12149ee656bf4f2525a425953a66acbda08c8c1 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 11 Dec 2024 00:45:40 +0000 Subject: [PATCH 20/27] Restyled by isort --- src/tools/push_av_server/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index 0561e63b559e1a..44f0b0a6f5df31 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -12,7 +12,7 @@ import sys import tempfile from pathlib import Path -from typing import Optional, Union, Literal +from typing import Literal, Optional, Union import uvicorn from cryptography import x509 From 4be0913272924801b5178a220a72cae45161b9c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 10 Dec 2024 17:23:55 -0800 Subject: [PATCH 21/27] Add support for the manifest upload --- src/tools/push_av_server/server.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index 44f0b0a6f5df31..d9c0010a76b301 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -477,9 +477,16 @@ def stream_files(dir: Path): return {"streams": streams} -# TODO app.put("/streams/{stream_id}") -# To conform to the ingest spec, support of that path with discarding the body -# and not doing anything. +@app.put("/streams/{stream_id}") +async def manifest_upload(stream_id: int, req: Request): + """The DASH manifest is uploaded onto the base path without any file path""" + + # Here we assume that no camera will upload an index.mpd file on their own. + # That is something that may not be true, in which case we would have to add + # another layer of abstraction on the file system where we can store the mpd + # file and the camera direct uploads. + return await segment_upload("index.mpd", stream_id, req) + @app.put("/streams/{stream_id}/{file_path:path}", status_code=202) async def segment_upload(file_path: str, stream_id: int, req: Request): @@ -531,13 +538,6 @@ async def segment_download(file_path: str, stream_id: int): return FileResponse(wd.path("streams", str(stream_id), file_path)) -# TODO app.post("streams/convert/{stream_id}") -# Will execute a ffmpeg conversion of the uploaded stream to a regular mp4 -# for easier consumption. Need to double check first if Firefox/Chrome can -# read DASH media with CMAF tracks. If they can no need to actually convert -# to a new format. - - @app.get("/certs", status_code=200) def list_certs(): server = [f.name for f in pathlib.Path(wd.path("certs", "server")).iterdir()] From 353f187511dce17981b58770283e60b99a5f4fad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 10 Dec 2024 17:24:09 -0800 Subject: [PATCH 22/27] fix variables in test --- src/python_testing/TC_PAVS_1_0.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/python_testing/TC_PAVS_1_0.py b/src/python_testing/TC_PAVS_1_0.py index e5b3610d410610..a7bf3f904ed7e5 100644 --- a/src/python_testing/TC_PAVS_1_0.py +++ b/src/python_testing/TC_PAVS_1_0.py @@ -141,8 +141,9 @@ async def test_wrong_cert_chain(self): wrong_chain = push_av_server.CAHierarchy(push_av_server.wd.mkdir("certs", "wrong"), "wrong chain", "client") wrong_chain.root_cert_path # Install onto the device - ["path_to_key", "path_to_cert"] = wrong_chain.gen_cert("dns", "csr") # Sign for the device + [path_to_key, path_to_cert] = wrong_chain.gen_cert("dns", "csr") # Sign for the device # Install onto the device + print(f"{path_to_cert}, {path_to_key}") # to not have unused values # After setting up the device, trigger an upload which will use the wrong cert and should fail From 8ce461d52154a242cd68be031a5e73343b1e5583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 10 Dec 2024 17:31:32 -0800 Subject: [PATCH 23/27] make index redirect to streams ui --- src/tools/push_av_server/server.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index d9c0010a76b301..85412ca1bc7830 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -21,7 +21,7 @@ from cryptography.hazmat.primitives.asymmetric.types import CertificateIssuerPrivateKeyTypes, CertificatePublicKeyTypes from cryptography.x509.oid import NameOID from fastapi import FastAPI, HTTPException, Request, Response -from fastapi.responses import FileResponse, HTMLResponse +from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from pydantic import BaseModel @@ -402,10 +402,9 @@ def gen_keypair(self, dns: str, override=False, duration: datetime.timedelta = d # UI website -@app.get("/", response_class=HTMLResponse) +@app.get("/", response_class=RedirectResponse) def root(): - with open("index.html", "r") as f: - return f.read() + return RedirectResponse("/ui/streams") @app.get("/ui/streams", response_class=HTMLResponse) From 4ecaca80b250dd2b1ef80c066b484bdf5a4718a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 10 Dec 2024 18:08:47 -0800 Subject: [PATCH 24/27] remove global variables --- src/python_testing/TC_PAVS_1_0.py | 2 + src/tools/push_av_server/server.py | 343 ++++++++++++++--------------- 2 files changed, 171 insertions(+), 174 deletions(-) diff --git a/src/python_testing/TC_PAVS_1_0.py b/src/python_testing/TC_PAVS_1_0.py index a7bf3f904ed7e5..f394caae9b21b9 100644 --- a/src/python_testing/TC_PAVS_1_0.py +++ b/src/python_testing/TC_PAVS_1_0.py @@ -10,6 +10,8 @@ class TC_PAVS_1_0(MatterBaseTest): for a better integration. It is not designed to be merged nor does it actually run. """ + # TODO Rewrite it to work with non-global state (e.g. decouple background server and setup) + def setup_class(self): super().setup_class() diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index 85412ca1bc7830..2c95f2c53e5a20 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -20,7 +20,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.asymmetric.types import CertificateIssuerPrivateKeyTypes, CertificatePublicKeyTypes from cryptography.x509.oid import NameOID -from fastapi import FastAPI, HTTPException, Request, Response +from fastapi import FastAPI, HTTPException, Request, Response, APIRouter from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates @@ -391,219 +391,210 @@ def gen_keypair(self, dns: str, override=False, duration: datetime.timedelta = d return (key_path, cert_bundle_path, False) -app = FastAPI() -app.mount("/static", StaticFiles(directory="static"), name="static") -templates = Jinja2Templates(directory="templates") -# Couldn't find how to do DI with state created in __main__ without using -# global variables, so using those. -wd: WorkingDirectory = None -device_hierarchy: CAHierarchy = None - - -# UI website - -@app.get("/", response_class=RedirectResponse) -def root(): - return RedirectResponse("/ui/streams") - - -@app.get("/ui/streams", response_class=HTMLResponse) -def ui_streams_list(request: Request): - s = list_streams() - return templates.TemplateResponse( - request=request, name="streams_list.html", context={"streams": s["streams"]} - ) - - -@app.get("/ui/streams/{stream_id}/{file_path:path}") -def ui_streams_details(request: Request, stream_id: int, file_path: str): - context = {} - context['streams'] = list_streams()['streams'] - context['stream_id'] = stream_id - context['file_path'] = file_path - - if file_path.endswith('.crt'): - context['type'] = 'cert' - p = wd.path("streams", str(stream_id), file_path) - with open(p, "r") as f: - context['cert'] = json.load(f) - else: - context['type'] = 'media' - context['probe'] = ffprobe_check(stream_id, file_path) - context['pretty_probe'] = json.dumps(context['probe'], sort_keys=True, indent=4) - - return templates.TemplateResponse(request=request, name="streams_details.html", context=context) - - -@app.get("/ui/certificates", response_class=HTMLResponse) -def ui_certificates_list(request: Request): - return templates.TemplateResponse( - request=request, name="certificates_list.html", context={"certs": list_certs()} - ) - - -@app.get("/ui/certificates/{hierarchy}/{name}", response_class=HTMLResponse) -def ui_certificates_details(request: Request, hierarchy: str, name: str): - context = certificate_details(hierarchy, name) - context["certs"] = list_certs() - - return templates.TemplateResponse(request=request, name="certificates_details.html", context=context) - -# APIs - - -@app.post("/streams", status_code=201) -def create_stream(): - # Find the last registered stream - dirs = [d for d in pathlib.Path(wd.path("streams")).iterdir() if d.is_dir()] - last_stream = int(dirs[-1].name) if len(dirs) > 0 else 0 - stream_id = last_stream + 1 - - wd.mkdir("streams", str(stream_id)) - - return {"stream_id": stream_id} +class SignClientCertificate(BaseModel): + """Request model to sign a client certificate""" + csr: str -@app.get("/streams") -def list_streams(): - dirs = [d for d in pathlib.Path(wd.path("streams")).iterdir() if d.is_dir()] +class PushAvServer: + + templates = Jinja2Templates(directory="templates") + + def __init__(self, wd: WorkingDirectory, device_hierarchy: CAHierarchy): + self.wd = wd + self.device_hierarchy = device_hierarchy + self.router = APIRouter() + + # UI + self.router.add_api_route("/", self.index, methods=["GET"], response_class=RedirectResponse) + self.router.add_api_route("/ui/streams", self.ui_streams_list, methods=["GET"], response_class=HTMLResponse) + self.router.add_api_route("/ui/streams/{stream_id}/{file_path:path}", self.ui_streams_details, methods=["GET"]) + self.router.add_api_route("/ui/certificates", self.ui_certificates_list, methods=["GET"], response_class=HTMLResponse) + self.router.add_api_route("/ui/certificates/{hierarchy}/{name}", + self.ui_certificates_details, methods=["GET"], response_class=HTMLResponse) + + # HTTP APIs + self.router.add_api_route("/streams", self.create_stream, methods=["POST"], status_code=201) + self.router.add_api_route("/streams", self.list_streams, methods=["GET"]) + self.router.add_api_route("/streams/probe/{stream_id}/{file_path:path}", self.ffprobe_check, methods=["GET"]) + self.router.add_api_route("/streams/{stream_id}", self.manifest_upload, methods=["PUT"]) + self.router.add_api_route("/streams/{stream_id}/{file_path:path}", self.segment_upload, methods=["PUT"], status_code=202) + self.router.add_api_route("/streams/{stream_id}/{file_path:path}", self.segment_download, methods=["GET"]) + self.router.add_api_route("/certs", self.list_certs, methods=["GET"], status_code=200) + self.router.add_api_route("/certs/{hierarchy}/{name}", self.certificate_details, methods=["GET"], status_code=200) + self.router.add_api_route("/certs/{name}/keypair", self.create_client_keypair, methods=["POST"]) + self.router.add_api_route("/certs/{name}/sign", self.sign_client_certificate, methods=["POST"]) + + # UI website + + def index(self): + return RedirectResponse("/ui/streams") + + def ui_streams_list(self, request: Request): + s = self.list_streams() + return self.templates.TemplateResponse( + request=request, name="streams_list.html", context={"streams": s["streams"]} + ) - def stream_files(dir: Path): - return [f.relative_to(dir) for f in dir.glob("**/*") if f.is_file()] + def ui_streams_details(self, request: Request, stream_id: int, file_path: str): + context = {} + context['streams'] = self.list_streams()['streams'] + context['stream_id'] = stream_id + context['file_path'] = file_path + + if file_path.endswith('.crt'): + context['type'] = 'cert' + p = self.wd.path("streams", str(stream_id), file_path) + with open(p, "r") as f: + context['cert'] = json.load(f) + else: + context['type'] = 'media' + context['probe'] = self.ffprobe_check(stream_id, file_path) + context['pretty_probe'] = json.dumps(context['probe'], sort_keys=True, indent=4) - streams = [{"id": d.name, "files": stream_files(d)} for d in dirs] + return self.templates.TemplateResponse(request=request, name="streams_details.html", context=context) - return {"streams": streams} + def ui_certificates_list(self, request: Request): + return self.templates.TemplateResponse( + request=request, name="certificates_list.html", context={"certs": self.list_certs()} + ) + def ui_certificates_details(self, request: Request, hierarchy: str, name: str): + context = self.certificate_details(hierarchy, name) + context["certs"] = self.list_certs() -@app.put("/streams/{stream_id}") -async def manifest_upload(stream_id: int, req: Request): - """The DASH manifest is uploaded onto the base path without any file path""" + return self.templates.TemplateResponse(request=request, name="certificates_details.html", context=context) - # Here we assume that no camera will upload an index.mpd file on their own. - # That is something that may not be true, in which case we would have to add - # another layer of abstraction on the file system where we can store the mpd - # file and the camera direct uploads. - return await segment_upload("index.mpd", stream_id, req) + # APIs + def create_stream(self): + # Find the last registered stream + dirs = [d for d in pathlib.Path(self.wd.path("streams")).iterdir() if d.is_dir()] + last_stream = int(dirs[-1].name) if len(dirs) > 0 else 0 + stream_id = last_stream + 1 -@app.put("/streams/{stream_id}/{file_path:path}", status_code=202) -async def segment_upload(file_path: str, stream_id: int, req: Request): - """Extract the parsed version of a client certificate. - See https://docs.python.org/3/library/ssl.html#ssl.SSLSocket.getpeercert - for the exact content. - """ - cert_details = req.scope["transport"].get_extra_info("ssl_object").getpeercert() + self.wd.mkdir("streams", str(stream_id)) - logging.debug(f"segment_upload. stream_id={stream_id} file_path:{file_path}") + return {"stream_id": stream_id} - if not wd.path("streams", str(stream_id)).exists(): - raise HTTPException(404, detail="Stream doesn't exists") + def list_streams(self): + dirs = [d for d in pathlib.Path(self.wd.path("streams")).iterdir() if d.is_dir()] - dst = wd.mkdir("streams", str(stream_id), file_path, is_file=True) + def stream_files(dir: Path): + return [f.relative_to(dir) for f in dir.glob("**/*") if f.is_file()] - with open(dst.with_suffix(dst.suffix + ".crt"), "w") as f: - f.write(json.dumps(cert_details)) + streams = [{"id": d.name, "files": stream_files(d)} for d in dirs] - with open(dst, "wb") as f: - async for chunk in req.stream(): - f.write(chunk) + return {"streams": streams} - return Response(status_code=202) + async def manifest_upload(self, stream_id: int, req: Request): + """The DASH manifest is uploaded onto the base path without any file path""" + # Here we assume that no camera will upload an index.mpd file on their own. + # That is something that may not be true, in which case we would have to add + # another layer of abstraction on the file system where we can store the mpd + # file and the camera direct uploads. + return await self.segment_upload("index.mpd", stream_id, req) -@app.get("/streams/probe/{stream_id}/{file_path:path}") -def ffprobe_check(stream_id: int, file_path: str): + async def segment_upload(self, file_path: str, stream_id: int, req: Request): + """Extract the parsed version of a client certificate. + See https://docs.python.org/3/library/ssl.html#ssl.SSLSocket.getpeercert + for the exact content. + """ + cert_details = req.scope["transport"].get_extra_info("ssl_object").getpeercert() - p = wd.path("streams", str(stream_id), file_path) + logging.debug(f"segment_upload. stream_id={stream_id} file_path:{file_path}") - if not p.exists(): - return HTTPException(404, detail="Stream doesn't exists") + if not self.wd.path("streams", str(stream_id)).exists(): + raise HTTPException(404, detail="Stream doesn't exists") - proc = subprocess.run( - ["ffprobe", "-show_streams", "-show_format", "-output_format", "json", str(p.absolute())], - capture_output=True - ) + dst = self.wd.mkdir("streams", str(stream_id), file_path, is_file=True) - if proc.returncode != 0: - # TODO Add more details (maybe stderr) to the response - return HTTPException(500) + with open(dst.with_suffix(dst.suffix + ".crt"), "w") as f: + f.write(json.dumps(cert_details)) - return json.loads(proc.stdout) + with open(dst, "wb") as f: + async for chunk in req.stream(): + f.write(chunk) + return Response(status_code=202) -@app.get("/streams/{stream_id}/{file_path:path}") -async def segment_download(file_path: str, stream_id: int): - return FileResponse(wd.path("streams", str(stream_id), file_path)) + def ffprobe_check(self, stream_id: int, file_path: str): + p = self.wd.path("streams", str(stream_id), file_path) -@app.get("/certs", status_code=200) -def list_certs(): - server = [f.name for f in pathlib.Path(wd.path("certs", "server")).iterdir()] - device = [f.name for f in pathlib.Path(wd.path("certs", "device")).iterdir()] + if not p.exists(): + return HTTPException(404, detail="Stream doesn't exists") - return {"server": server, "device": device} + proc = subprocess.run( + ["ffprobe", "-show_streams", "-show_format", "-output_format", "json", str(p.absolute())], + capture_output=True + ) + if proc.returncode != 0: + # TODO Add more details (maybe stderr) to the response + return HTTPException(500) -@app.get("/certs/{hierarchy}/{name}", status_code=200) -def certificate_details(hierarchy: str, name: str): - data = pathlib.Path(wd.path("certs", hierarchy, name)).read_bytes() - type = "key" if name.endswith(".key") else "cert" + return json.loads(proc.stdout) - key = None - cert = None - if type == "key": - key = serialization.load_pem_private_key(data, None) - key = { - "key_size": key.key_size, - "private_key": key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - ), - "public_key": key.public_key().public_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PublicFormat.PKCS1, - ), - } - else: - cert = x509.load_pem_x509_certificate(data) - cert = { - "public_cert": cert.public_bytes(serialization.Encoding.PEM), - "serial_number": hex(cert.serial_number), - "not_valid_before": cert.not_valid_before_utc, - "not_valid_after": cert.not_valid_after_utc, - # public_key? fingerprint? - "issuer": cert.issuer.rfc4514_string(), - "subject": cert.subject.rfc4514_string(), - "extensions": [str(ext) for ext in cert.extensions] - } + async def segment_download(self, file_path: str, stream_id: int): + return FileResponse(self.wd.path("streams", str(stream_id), file_path)) - return {"type": type, "key": key, "cert": cert} + def list_certs(self): + server = [f.name for f in pathlib.Path(self.wd.path("certs", "server")).iterdir()] + device = [f.name for f in pathlib.Path(self.wd.path("certs", "device")).iterdir()] + return {"server": server, "device": device} -@app.post("/certs/{name}/keypair") -def create_client_keypair(name: str, override: bool = True): - (key, cert, created) = device_hierarchy.gen_keypair(name, override) + def certificate_details(self, hierarchy: str, name: str): + data = pathlib.Path(self.wd.path("certs", hierarchy, name)).read_bytes() + type = "key" if name.endswith(".key") else "cert" - return {key, cert, created} + key = None + cert = None + if type == "key": + key = serialization.load_pem_private_key(data, None) + key = { + "key_size": key.key_size, + "private_key": key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ), + "public_key": key.public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.PKCS1, + ), + } + else: + cert = x509.load_pem_x509_certificate(data) + cert = { + "public_cert": cert.public_bytes(serialization.Encoding.PEM), + "serial_number": hex(cert.serial_number), + "not_valid_before": cert.not_valid_before_utc, + "not_valid_after": cert.not_valid_after_utc, + # public_key? fingerprint? + "issuer": cert.issuer.rfc4514_string(), + "subject": cert.subject.rfc4514_string(), + "extensions": [str(ext) for ext in cert.extensions] + } + return {"type": type, "key": key, "cert": cert} -class SignClientCertificate(BaseModel): - csr: str + def create_client_keypair(self, name: str, override: bool = True): + (key, cert, created) = self.device_hierarchy.gen_keypair(name, override) + return {key, cert, created} -@app.post("/certs/{name}/issue") -def sign_client_certificate( - name: str, req: SignClientCertificate, override: bool = True -): - (key, cert, created) = device_hierarchy.gen_cert(name, req.csr, override) + def sign_client_certificate( + self, name: str, req: SignClientCertificate, override: bool = True + ): + (key, cert, created) = self.device_hierarchy.gen_cert(name, req.csr, override) - return {key, cert, created} + return {key, cert, created} def run(host: Optional[str], port: Optional[int], working_directory: Optional[str], dns: Optional[str]): - global wd, device_hierarchy """Run the reference server. This function will not return. In the context where a background server is required, the multiprocessing.Process object can be used. @@ -611,7 +602,6 @@ def run(host: Optional[str], port: Optional[int], working_directory: Optional[st with WorkingDirectory(working_directory) as directory: # Sharing state with the various endpoints - wd = directory dns = "localhost" if dns is None else f"{dns}._http._tcp_.local." @@ -636,9 +626,14 @@ def run(host: Optional[str], port: Optional[int], working_directory: Optional[st zeroconf.register_service(svc_info) # Streams holder - wd.mkdir("streams") + directory.mkdir("streams") + + app = FastAPI() + app.mount("/static", StaticFiles(directory="static"), name="static") + pas = PushAvServer() + app.include_router(pas.router) - # Setup the web server + # Start the web server try: uvicorn.run( app, From 89110fc591d754be35fc14c2403bc7db18f79a25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Thu, 12 Dec 2024 15:44:18 -0800 Subject: [PATCH 25/27] Make the push av server runnable in the background --- src/python_testing/TC_PAVS_1_0.py | 26 ++----- src/tools/push_av_server/README.md | 2 +- src/tools/push_av_server/server.py | 111 +++++++++++++++++------------ 3 files changed, 73 insertions(+), 66 deletions(-) diff --git a/src/python_testing/TC_PAVS_1_0.py b/src/python_testing/TC_PAVS_1_0.py index f394caae9b21b9..8bdb7e3454db97 100644 --- a/src/python_testing/TC_PAVS_1_0.py +++ b/src/python_testing/TC_PAVS_1_0.py @@ -1,5 +1,3 @@ -from multiprocessing import Process - import push_av_server from matter_testing_support import MatterBaseTest, TestStep, async_test_body, default_matter_test_main @@ -7,22 +5,18 @@ class TC_PAVS_1_0(MatterBaseTest): """ NOTE: this class is only a guide to understand what APIs I'd need to integrate in the push av server - for a better integration. It is not designed to be merged nor does it actually run. + for a better integration. It is not designed to be merged nor does it actually run (no chip module). """ - # TODO Rewrite it to work with non-global state (e.g. decouple background server and setup) - def setup_class(self): super().setup_class() - self.proc = Process(target=push_av_server.run, - args=("127.0.0.1", 1234, None, "localhost"), - daemon=True) - self.proc.start() + self.av_ctx = push_av_server.PushAvContext("127.0.0.1", 1234, None, "localhost") + self.av_ctx.start_in_background() def teardown_class(self): super().teardown_class() - self.proc.terminate() + self.av_ctx.terminate() def steps_TC_PAVS_1_0(self): return [TestStep(1, "Commissioning, already done", is_commissioning=True), @@ -36,15 +30,12 @@ def steps_TC_PAVS_1_0(self): @async_test_body async def test_TC_PAVS_1_0(self): - srv = push_av_server.start("localhost", 1234) - srv.run_in_thread() - # commissioning - already done self.step(1) self.step(2) # Access CA cert via the push_av_server package. - push_av_server.device_hierarchy.root_cert + print(self.av_ctx.device_hierarchy.root_cert) # read TLSCertificateManagament attributes to validate state # Send the TLSCertificateManagament.ProvisionRootCertificate command # Assert we got a response that contains a CA id @@ -55,7 +46,7 @@ async def test_TC_PAVS_1_0(self): self.step("3b") # Generate nonce # send TLSCertificateManagement.TLSClientCSR, receive TLSClientCSRResponse - push_av_server.device_hierarchy.gen_cert("device name", "csr") + print(self.av_ctx.device_hierarchy.gen_cert("device name", "csr")) # send ProvisionClientCertificate, receive ProvisionClientCertificateResponse self.step(4) @@ -133,13 +124,8 @@ async def test_TC_PAVS_1_0(self): self.step(7) # TBD. deallocation logic - srv.stop() - async def test_wrong_cert_chain(self): """Intended as an example of how to install an unrecognised """ - srv = push_av_server.start("localhost", 1234) - srv.run_in_thread() - wrong_chain = push_av_server.CAHierarchy(push_av_server.wd.mkdir("certs", "wrong"), "wrong chain", "client") wrong_chain.root_cert_path # Install onto the device diff --git a/src/tools/push_av_server/README.md b/src/tools/push_av_server/README.md index 9e2c4927e1f4ee..374ac7f30dab57 100644 --- a/src/tools/push_av_server/README.md +++ b/src/tools/push_av_server/README.md @@ -35,5 +35,5 @@ $ curl -XGET --cacert ~/.pavstest/certs/server/root.pem https://localhost:1234/s # Get detailed information about the uploaded media file. # This correspond to the ffprobe tool output -$ curl --cacert ~/.pavstest/certs/server/root.pem -XGET 'https://localhost:1234/streams/probe/1/cmaf/example/video-720p.cmfv' +$ curl --cacert ~/.pavstest/certs/server/root.pem -XGET 'https://localhost:1234/probe/1/cmaf/example/video-720p.cmfv' ``` diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index 2c95f2c53e5a20..b3ae237fdd9c22 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -13,6 +13,7 @@ import tempfile from pathlib import Path from typing import Literal, Optional, Union +import multiprocessing import uvicorn from cryptography import x509 @@ -63,6 +64,9 @@ def __enter__(self): return self def __exit__(self, exc, value, tb): + self.cleanup() + + def cleanup(self): if self.tmp: self.tmp.cleanup() @@ -594,65 +598,78 @@ def sign_client_certificate( return {key, cert, created} -def run(host: Optional[str], port: Optional[int], working_directory: Optional[str], dns: Optional[str]): - """Run the reference server. This function will not return. - In the context where a background server is required, the multiprocessing.Process object - can be used. - """ - with WorkingDirectory(working_directory) as directory: - - # Sharing state with the various endpoints +class PushAvContext: + """Hold the context for a full Push AV Server including temporary disk, CA hierarchies and web server""" - dns = "localhost" if dns is None else f"{dns}._http._tcp_.local." + def __init__(self, host: Optional[str], port: Optional[int], working_directory: Optional[str], dns: Optional[str]): + self.directory = WorkingDirectory(working_directory) + self.host = host + self.port = port + self.dns = "localhost" if dns is None else f"{dns}._http._tcp_.local." + self.proc: multiprocessing.Process | None = None # Create CA hierarchies (for webserver and devices) - device_hierarchy = CAHierarchy(directory.mkdir("certs", "device"), "device", "client") - server_hierarchy = CAHierarchy(directory.mkdir("certs", "server"), "server", "server") - (server_key_file, server_cert_file, _) = server_hierarchy.gen_keypair(dns, override=True) + self.device_hierarchy = CAHierarchy(self.directory.mkdir("certs", "device"), "device", "client") + self.server_hierarchy = CAHierarchy(self.directory.mkdir("certs", "server"), "server", "server") + (self.server_key_file, self.server_cert_file, _) = self.server_hierarchy.gen_keypair(self.dns, override=True) # mDNS configuration. Registration only happen if the dns isn't localhost. - zeroconf = Zeroconf() - svc_info = None + self.zeroconf = Zeroconf() + self.svc_info = None - if dns != "localhost": - svc_info = ServiceInfo( + if self.dns != "localhost": + self.svc_info = ServiceInfo( "_http._tcp.local.", - name=dns, + name=self.dns, addresses=[socket.inet_aton("127.0.0.1")], port=1234, ) - # Advertise over mDNS - logging.info("Advertising the service as %s", svc_info) - zeroconf.register_service(svc_info) # Streams holder - directory.mkdir("streams") - - app = FastAPI() - app.mount("/static", StaticFiles(directory="static"), name="static") - pas = PushAvServer() - app.include_router(pas.router) - - # Start the web server - try: - uvicorn.run( - app, - host=host, - port=port, - ssl_keyfile=server_key_file, - ssl_certfile=server_cert_file, - ssl_cert_reqs=ssl.CERT_OPTIONAL, - ssl_ca_certs=device_hierarchy.root_cert_path, - ) - finally: - if dns != "localhost": - zeroconf.unregister_service(svc_info) + self.directory.mkdir("streams") + + self.app = FastAPI() + self.app.mount("/static", StaticFiles(directory="static"), name="static") + pas = PushAvServer(self.directory, self.device_hierarchy) + self.app.include_router(pas.router) + + def start_in_background(self): + if self.proc: + logging.warning("Attempting to start a server when one is already running, no new server is being started.") + return + + # Advertise over mDNS + if self.svc_info: + logging.info("Advertising the service as %s", self.svc_info) + self.zeroconf.register_service(self.svc_info) + + def background_job(): + # Start the web server + try: + uvicorn.run( + self.app, + host=self.host, + port=self.port, + ssl_keyfile=self.server_key_file, + ssl_certfile=self.server_cert_file, + ssl_cert_reqs=ssl.CERT_OPTIONAL, + ssl_ca_certs=self.device_hierarchy.root_cert_path, + ) + finally: + if self.svc_info: + self.zeroconf.unregister_service(self.svc_info) - # directory.print_tree() + # Spawning the function results in python not being able to pickle the full context + # (most notably cryptography's rust bindings). So instead we force use forks as the + # way to create processes. + multiprocessing.set_start_method('fork') + self.proc = multiprocessing.Process(target=background_job, daemon=True) + self.proc.start() + def terminate(self): + self.proc.terminate() + self.directory.cleanup() -# TODO UI -# html page to provide a good way to see uploaded content (file "browser" + video player) if __name__ == "__main__": logging.basicConfig( @@ -679,4 +696,8 @@ def run(host: Optional[str], port: Optional[int], working_directory: Optional[st args = parser.parse_args() - run(args.host, args.port, args.working_directory, args.dns) + ctx = PushAvContext(args.host, args.port, args.working_directory, args.dns) + ctx.start_in_background() + print(ctx.proc) + ctx.proc.join() + ctx.terminate() From 9c854e6f3b1fcfa87a6e58186b4750ec20636a90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Thu, 12 Dec 2024 16:21:22 -0800 Subject: [PATCH 26/27] Document (and fix) certificate sign via API --- src/tools/push_av_server/README.md | 11 +++++++++++ src/tools/push_av_server/server.py | 6 +++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/tools/push_av_server/README.md b/src/tools/push_av_server/README.md index 374ac7f30dab57..b64b17cbce643d 100644 --- a/src/tools/push_av_server/README.md +++ b/src/tools/push_av_server/README.md @@ -36,4 +36,15 @@ $ curl -XGET --cacert ~/.pavstest/certs/server/root.pem https://localhost:1234/s # Get detailed information about the uploaded media file. # This correspond to the ffprobe tool output $ curl --cacert ~/.pavstest/certs/server/root.pem -XGET 'https://localhost:1234/probe/1/cmaf/example/video-720p.cmfv' + +# You can also use the web server to sign certificates if given a CSR. +# First create a key and csr for your device: +$ openssl req -new -newkey rsa:2048 -nodes -keyout client.key -out client.csr -subj "/CN=test" + +# When sending the CSR over JSON we need to have the newline characters be the literal \n. +$ sed '$!G' client.csr | paste -sd '\\n' - > client.curl.csr + +# Then sign it with the server +$ curl --cacert ~/.pavstest/certs/server/root.pem -XPOST 'https://localhost:1234/certs/my-device/sign' -d "{\"csr\":\"$(cat client.curl.csr)\"}" --header "content-type: application/json" + ``` diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index b3ae237fdd9c22..4206b2ba9008fe 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -248,7 +248,7 @@ def _save_cert( turn make it very unsecure. """ cert_path = self.directory / f"{name}.pem" - key_path = self.directory / f"{name}.key" + key_path = self.directory / f"{name}.key" if key else None if key: with open(key_path, "wb") as f: @@ -344,7 +344,7 @@ def gen_cert(self, dns: str, csr: str, override=False, duration: datetime.timede Generate a certificate signed by this CA hierarchy using the provided CSR. Returns the path to the key, cert, and whether it was reused or not. """ - signing_request = x509.load_pem_x509_csr(csr) + signing_request = x509.load_pem_x509_csr(csr.encode('utf-8')) signing_request.public_key() # If we don't always override, first check if an existing keypair already exists @@ -356,7 +356,7 @@ def gen_cert(self, dns: str, csr: str, override=False, duration: datetime.timede return (key_path, cert_path, True) # Sign certificate - cert = self._sign_cert(dns, csr.public_key(), duration) + cert = self._sign_cert(dns, signing_request.public_key(), duration) # Save that information to disk (key_path, cert_bundle_path) = self._save_cert( From 0ee9316c71d6bd342d57b4d09b03f13479388772 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 13 Dec 2024 00:22:22 +0000 Subject: [PATCH 27/27] Restyled by isort --- src/tools/push_av_server/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/push_av_server/server.py b/src/tools/push_av_server/server.py index 4206b2ba9008fe..dc2621ab5c3b4f 100644 --- a/src/tools/push_av_server/server.py +++ b/src/tools/push_av_server/server.py @@ -2,6 +2,7 @@ import datetime import json import logging +import multiprocessing import os.path import pathlib import random @@ -13,7 +14,6 @@ import tempfile from pathlib import Path from typing import Literal, Optional, Union -import multiprocessing import uvicorn from cryptography import x509 @@ -21,7 +21,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.asymmetric.types import CertificateIssuerPrivateKeyTypes, CertificatePublicKeyTypes from cryptography.x509.oid import NameOID -from fastapi import FastAPI, HTTPException, Request, Response, APIRouter +from fastapi import APIRouter, FastAPI, HTTPException, Request, Response from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates