diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 953650e..256c763 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,8 @@ jobs: - uses: actions/checkout@v4 - name: Install dependencies run: | - apt-get update && apt-get install --yes --no-install-recommends make openssl + apt-get update && apt-get install --yes --no-install-recommends make openssl python3 python3-poetry + poetry install --no-ansi - name: Verify ruleset signature run: | make verify diff --git a/.gitignore b/.gitignore index 9e83c7e..5f192d0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ test-key.jwk public.pem # Generated files -rulesets/default.rulesets rulesets/default.rulesets.json # Byte-compiled / optimized / DLL files diff --git a/Makefile b/Makefile index 6716753..4f19d91 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ image := fpf.local/securedrop-https-everywhere-ruleset:$(shell cat latest-rulesets-timestamp) -DEFAULT_GOAL: rules +DEFAULT_GOAL: help .PHONY: check-black check-black: ## Check Python source code formatting with black @@ -16,9 +16,19 @@ test-key: ## Generates a test key for development/testing purposes locally. openssl rsa -in key.pem -outform PEM -pubout -out public.pem poetry run python jwk.py > test-key.jwk -.PHONY: rules -rules: ## Regenerates rulesets in preparation for signing ceremony - poetry run ./scripts/generate-and-sign +.PHONY: generate +generate: ## Regenerates rulesets in preparation for signing ceremony + echo "Generating SecureDrop Onion Name rulesets..." + poetry run python3 sddir.py + poetry run python3 upstream/merge-rulesets.py --source_dir rulesets + +.PHONY: sign +sign: ## Signs the latest ruleset + echo "Preparing rulesets for airgapped signature request..." + ./upstream/async-request.sh public_release.pem . + echo "Updating index for SecureDrop rules..." + ./update_index.sh + echo "Finished. Please review local changes, and commit as appropriate." .PHONY: serve serve: ## Builds Nginx container to serve generated files @@ -32,6 +42,7 @@ serve: ## Builds Nginx container to serve generated files verify: ## Verifies the signature of the latest ruleset. Requires openssl to be installed. @echo "Attempting to verify ruleset signature using openssl." @./scripts/verify + @poetry run pytest -v .PHONY: help help: diff --git a/README.md b/README.md index ff63527..76074a3 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # HTTPS-Everywhere Rulesets for SecureDrop -`securedrop-https-everywhere-ruleset` is used to create a signed HTTPS Everywhere ruleset that maps full-length .onion addresses to user-friendly [onion names](https://securedrop.org/faq/getting-onion-name-your-securedrop/) for some news organizations listed in the [SecureDrop directory](https://securedrop.org/directory/). Any time a new onion name is approved, we add its mapping to our HTTPS Everywhere ruleset and deploy it to https://securedrop.org/https-everywhere/ . Tor Browser automatically includes our ruleset in the default HTTPS Everywhere extension and checks for updates on startup. (Tor Browser will soon switch to checking https://securedrop.org/https-everywhere-2021/ which uses our new release signing key). +`securedrop-https-everywhere-ruleset` is used to create a signed HTTPS Everywhere ruleset that maps full-length .onion addresses to user-friendly [onion names](https://securedrop.org/faq/getting-onion-name-your-securedrop/) for some news organizations listed in the [SecureDrop directory](https://securedrop.org/directory/). Any time a new onion name is approved, we add its mapping to our HTTPS Everywhere ruleset and deploy it to https://securedrop.org/https-everywhere-2021/ . Tor Browser automatically includes our ruleset in the default HTTPS Everywhere extension and checks for updates on startup. ## Development @@ -24,13 +24,11 @@ which will create `test-key.jwk` in your current working directory. 2. Add their domain name and the requested URL to the `onboarded.txt` via PR into this repository. We match the domain based on the landing page of the organization, comparing the `netloc` in a URL with structure `scheme://netloc/path;parameters?query#fragment`. -3. Next, generate and sign the update ruleset using the following command (requires signing key, please ping a key holder for assistance): +3. Next, generate the updated ruleset with `make generate` and review the output. -``` -make rules -``` +4. Once satisfied, you can sign it with `make sign` (requires signing key, please ping a key holder for assistance). -4. Commit all files generated by the script above and open a Pull Request to this repository. Once the PR is merged, the rulesets will automatically be deployed to production. +5. Commit all files generated by the script above and open a Pull Request to this repository. Once the PR is merged, the rulesets will automatically be deployed to production. ## Verifying changes diff --git a/poetry.lock b/poetry.lock index a0f0573..7f7185b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "authlib" @@ -335,6 +335,17 @@ files = [ [package.extras] all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "mypy-extensions" version = "1.0.0" @@ -398,6 +409,21 @@ docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-a test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"] type = ["mypy (>=1.11.2)"] +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + [[package]] name = "pyasn1" version = "0.6.1" @@ -420,6 +446,26 @@ files = [ {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] +[[package]] +name = "pytest" +version = "8.3.3" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, + {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2" + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + [[package]] name = "requests" version = "2.32.3" @@ -461,4 +507,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = ">= 3.11" -content-hash = "eef11cd99663782e8179aeb17de6daa2b78aa75fb905a645260e88721b80f8b8" +content-hash = "9383819fb8219250f39c527368ee1c0a2a5d1cc61ef60f234740029a73928341" diff --git a/pyproject.toml b/pyproject.toml index 7d1755c..420c9fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ pgpy = ">=0.6.0" [tool.poetry.group.dev.dependencies] black = "*" +pytest = "^8.3.3" [tool.black] line-length = 100 diff --git a/rulesets/default.rulesets b/rulesets/default.rulesets new file mode 100644 index 0000000..beb051d --- /dev/null +++ b/rulesets/default.rulesets @@ -0,0 +1 @@ +[{"name":"2600: The Hacker Quarterly","target":["2600.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://2600.securedrop.tor.onion","to":"http://cy6wj77vryhcyh6go576hxycjz4wxlo4s5vevdinkw3armwzty5jozyd.onion"}]},{"name":"ABC","target":["abc.au.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://abc.au.securedrop.tor.onion","to":"http://dqa4zahticcobfq5rmmmbewbdtyiznbl75hu23k4i37y7yfoosrh7mqd.onion"}]},{"name":"Aftenposten AS","target":["aftenposten.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://aftenposten.securedrop.tor.onion","to":"http://tiykfvhb562gheutfnedysnhrxpxoztyszkqyroloyepwzxmxien77id.onion"}]},{"name":"Aftonbladet","target":["aftonbladet.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://aftonbladet.securedrop.tor.onion","to":"http://xm33ge4kupk5o66eqxcd2r4fqcplpqb2sbdduf5z2nw4g2jrxe57luid.onion"}]},{"name":"Al Jazeera Media Network","target":["ajiunit.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://ajiunit.securedrop.tor.onion","to":"http://jkta32w5gvk6pmqdfwj67psojot3l2iwoqbdvrvywi5bkudfeandq7id.onion"}]},{"name":"Apache","target":["apache.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://apache.securedrop.tor.onion","to":"http://okd7utbak43lm7qaixr6yv7s62e32mhngjsfpjn26eklokqofg6776yd.onion"}]},{"name":"Barton Gellman","target":["bartongellman.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://bartongellman.securedrop.tor.onion","to":"http://hxywmnvdz5f2l5gqwjfcejdpla7nhj35dn5cf5l6qevjb77wasnna3qd.onion"}]},{"name":"Bloomberg Industry Group","target":["bloombergindustrygroup.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://bloombergindustrygroup.securedrop.tor.onion","to":"http://33buewrpzrfpttl7kerqvtvzyo3ivumilwwmeqjryzajusltibaqc6ad.onion"}]},{"name":"Bloomberg Law","target":["bloomberglaw.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://bloomberglaw.securedrop.tor.onion","to":"http://33buewrpzrfpttl7kerqvtvzyo3ivumilwwmeqjryzajusltibaqc6ad.onion"}]},{"name":"Bloomberg News","target":["bloomberg.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://bloomberg.securedrop.tor.onion","to":"http://ogdwaroarq4p6rnfn2hl4crvldyruyc2g24435qtxmd3twhevg7dsqid.onion"}]},{"name":"CBC","target":["cbcrc.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://cbcrc.securedrop.tor.onion","to":"http://gppg43zz5d2yfuom3yfmxnnokn3zj4mekt55onlng3zs653ty4fio6qd.onion"}]},{"name":"The Center for Public Integrity","target":["publicintegrity.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://publicintegrity.securedrop.tor.onion","to":"http://ahgpmkiaqfde4innkotgz5q6bgt4gbxmelqod3tjtmpdt3zvxaxareyd.onion"}]},{"name":"Claudio Guarnieri","target":["nex.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://nex.securedrop.tor.onion","to":"http://7dw7foypguycptlodmkscnziw5a65ilivzz6ajiei3yhe3gsfojlqwad.onion"}]},{"name":"CNN","target":["cnn.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://cnn.securedrop.tor.onion","to":"http://qmifwf762qftydprw2adbg7hs2mkunac5xrz3cb5busaflji3rja5lid.onion"}]},{"name":"Dagbladet","target":["dagbladet.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://dagbladet.securedrop.tor.onion","to":"http://ydbpz5knb6ji3bdtahhm3wo7sed6lsy5vqnwfpnhpez4bquvoexbz7qd.onion"}]},{"name":"Der Spiegel","target":["spiegel.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://spiegel.securedrop.tor.onion","to":"http://q6vdlj2ukulrqk37piqgxucpcwtxzdjhvjzqrfbevuhrzimsgjltmpqd.onion"}]},{"name":"Disclose","target":["disclose.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://disclose.securedrop.tor.onion","to":"http://3tcbrdg2ejwu5nzbjg7xqixkis6mdbgkkthcyxmzv2q3oi6v7th5ahqd.onion"}]},{"name":"DR - Danish Broadcasting Corporation","target":["dr.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://dr.securedrop.tor.onion","to":"http://hpaauqmv2wegiu4cz6st6hty4s7gwqol272xhcu3xmh6azw2f2zffgid.onion"}]},{"name":"Espen Andersen","target":["espena.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://espena.securedrop.tor.onion","to":"http://tsovw443sbbaizc3mxwuqrnbc4uiml3x3uuinmplthsmpiqdphl7v5yd.onion"}]},{"name":"Financial Times","target":["ft.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://ft.securedrop.tor.onion","to":"http://nqu6crmtnzs2hs5abo2uqni53yqsnnwqnerdxuzyz5yxairxlzjzt6yd.onion"}]},{"name":"Forbes","target":["forbes.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://forbes.securedrop.tor.onion","to":"http://6zonlfhh7aqtfwoyvdlad3nxn6ljecx2k6tyyy3spt43nn54q6lvncid.onion"}]},{"name":"Forbidden Stories","target":["forbiddenstories.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://forbiddenstories.securedrop.tor.onion","to":"http://fg25fqpu2dnxp24xs3jlcley4hp2inshpzek44q3czkhq3zffoqk26id.onion"}]},{"name":"The Globe and Mail","target":["theglobeandmail.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://theglobeandmail.securedrop.tor.onion","to":"http://a4zum5ydurvljrohxqp2rjjal5kro4ge2q2qizuonf2jubkhcr627gad.onion"}]},{"name":"Greekleaks","target":["greekleaks.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://greekleaks.securedrop.tor.onion","to":"http://jatasaqcoe7lqdpcyxo7vl3e5tdvl5jgmtadfat77i25qdj6z6a4ulad.onion"}]},{"name":"The Guardian","target":["theguardian.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://theguardian.securedrop.tor.onion","to":"http://xp44cagis447k3lpb4wwhcqukix6cgqokbuys24vmxmbzmaq2gjvc2yd.onion"}]},{"name":"HuffPost","target":["huffpost.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://huffpost.securedrop.tor.onion","to":"http://ppw2pmtagxykinex6uubypsommtrcg6ytdh6bcr6agq2wxnrweao4cad.onion"}]},{"name":"Institute for Quantitative Social Science at Harvard University","target":["iqss.harvard.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://iqss.harvard.securedrop.tor.onion","to":"http://5kcyaqagvnrvyan7y5ntzreqsn2msowqlmtoo46qju2pctlbkzzztxqd.onion"}]},{"name":"The Intercept","target":["theintercept.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://theintercept.securedrop.tor.onion","to":"http://lhollo6vzrft3w77mgm67fhfv3fjadmf7oinmafa7tbmupc273oi7kid.onion"}]},{"name":"Investigace.cz","target":["investigace.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://investigace.securedrop.tor.onion","to":"http://e2kkexl7exz6rg7fhl4oftkaeojm7wlbw567hqu2tbrjlixsjjoynzad.onion"}]},{"name":"K-Tipp","target":["ktipp.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://ktipp.securedrop.tor.onion","to":"http://tukldpfzdizsrfyvdljnipmvix2dcb5hmfoemcidkw7bq56wxblk6did.onion"}]},{"name":"Kenneth R. Rosen","target":["kennethrrosen.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://kennethrrosen.securedrop.tor.onion","to":"http://dpsw5tvlh2pccviydqw2cz5tjszd34zcdj322oikydqvgsqwitxup7yd.onion"}]},{"name":"Lessig.law LLC","target":["lessig.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://lessig.securedrop.tor.onion","to":"http://o4nhtigrvss5wktskr5ph5m22ewmhk7nr5at2tac2wdsworcqz62vsqd.onion"}]},{"name":"New York Times","target":["nytimes.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://nytimes.securedrop.tor.onion","to":"https://ej3kv4ebuugcmuwxctx5ic7zxh73rnxt42soi3tdneu2c2em55thufqd.onion"}]},{"name":"News24","target":["news24.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://news24.securedrop.tor.onion","to":"http://uhmj4j5pnwbpmkebfze3qgjmkum465fvok376nxtpku5yvyv5takz6qd.onion"}]},{"name":"NOYB","target":["noyb.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://noyb.securedrop.tor.onion","to":"http://xjc4s5z26i2z5tzjzj3w6jwzuomedzsahq4tccktwdcs6fldt4ojznqd.onion"}]},{"name":"NRK","target":["nrk.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://nrk.securedrop.tor.onion","to":"http://537ztcntpbmspja4mkpxldpsoc46mqlssnsaklqnfw3gnlpj5glcjgid.onion"}]},{"name":"POLITICO","target":["politico.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://politico.securedrop.tor.onion","to":"http://mzi5yynpd6qqq3lnh7vnaojy36v3hcorytsut47zwkguhnorduyxwead.onion"}]},{"name":"ProPublica","target":["propublica.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://propublica.securedrop.tor.onion","to":"http://lvtu6mh6dd6ynqcxtd2mseqfkm7g2iuxvjobbyzpgx2jt427zvd7n3ad.onion"}]},{"name":"San Francisco Chronicle","target":["sfchronicle.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://sfchronicle.securedrop.tor.onion","to":"http://b52gknakgsyqqeq476oi5nymw6yapysfig4owqgwppi5qpuk4az6bxad.onion"}]},{"name":"Stavanger Aftenblad","target":["aftenbladet.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://aftenbladet.securedrop.tor.onion","to":"http://4beybcv5e7xya4xu2nzdqkohawm32imugjtatkvmp2xwgfhcoj64slid.onion"}]},{"name":"Stefania Maurizi","target":["maurizi.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://maurizi.securedrop.tor.onion","to":"http://jxsb4ovmavjy3r64bak4ha63xwggf3nzf3vikvs23r2avm5rhzmaqtqd.onion"}]},{"name":"Suddeutsche Zeitung","target":["sueddeutsche.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://sueddeutsche.securedrop.tor.onion","to":"http://udhauo3m3fh7v6yfiuornjzxn3fh6vlp4ooo3wogvghcnv5xik6mnayd.onion"}]},{"name":"Taz","target":["taz.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://taz.securedrop.tor.onion","to":"http://tazleakssvtc2lqrhkpvbzo6qwolcldzkzoexo7wombufd6a573bhlid.onion"}]},{"name":"TechCrunch","target":["techcrunch.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://techcrunch.securedrop.tor.onion","to":"http://vplxle7awnyvvvduv6exnwrxbf4gzsh7lv7fxosnfl2ecidkttcbfcqd.onion"}]},{"name":"The Economist","target":["theeconomist.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://theeconomist.securedrop.tor.onion","to":"http://mxmddqsh4jnr4gjan37ayin3fu5ecnejxge4wjhj4i45qq5djbxdjtad.onion"}]},{"name":"Thomson Reuters","target":["reuters.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://reuters.securedrop.tor.onion","to":"http://dvvbik7vtmvwwgj2cziqa36noa26l2pweghd26e5l5qwdnqtwmfhz5id.onion"}]},{"name":"Toronto Star","target":["torontostar.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://torontostar.securedrop.tor.onion","to":"http://yj3b7rgmglcocbbvzrwfbo4d6j2aa7thwupra4yqutbd27v3vxcpvgid.onion"}]},{"name":"TV2 Denmark","target":["tv2.dk.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://tv2.dk.securedrop.tor.onion","to":"http://srumyob2jq5nvppzt66aaab333n2wmq6xgkg4khfe24ixdb7umf7mtyd.onion"}]},{"name":"The Washington Post","target":["washingtonpost.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://washingtonpost.securedrop.tor.onion","to":"https://vfnmxpa6fo4jdpyq3yneqhglluweax2uclvxkytfpmpkp5rsl75ir5qd.onion"}]},{"name":"Whistleblower Aid","target":["whistlebloweraid.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://whistlebloweraid.securedrop.tor.onion","to":"http://kogbxf4ysay2qzozmg7ar45ijqmj2vxrwqa4upzqq2i7sqj7wv7wcdqd.onion"}]}] \ No newline at end of file diff --git a/scripts/generate-and-sign b/scripts/generate-and-sign deleted file mode 100755 index 2839232..0000000 --- a/scripts/generate-and-sign +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Utility script to generate the SecureDrop HTTPS Everywhere rulesets, -# used for managing Onion Names for SecureDrop instances. -# -# Much of the business logic is taken verbatim from the EFF HTTPSE repo: -# -# https://github.com/EFForg/https-everywhere/blob/master/docs/en_US/ruleset-update-channels.md#signing -# -set -euo pipefail - -# Generate the SD rulesets -echo "Generating SecureDrop Onion Name rulesets..." -python3 sddir.py - -python3 upstream/merge-rulesets.py --source_dir rulesets -echo "Preparing rulesets for airgapped signature request..." -./upstream/async-request.sh public_release.pem . - -echo "Updating index for SecureDrop rules..." -./update_index.sh - -echo "Finished. Please review local changes, and commit as appropriate." -# TODO: Not automatically running 'git add *' due to -# https://github.com/freedomofpress/securedrop-https-everywhere-ruleset/issues/20 diff --git a/test_rulesets.py b/test_rulesets.py new file mode 100644 index 0000000..e11a9c0 --- /dev/null +++ b/test_rulesets.py @@ -0,0 +1,57 @@ +import gzip +import json +import xml.etree.ElementTree +from pathlib import Path + +LATEST_TIMESTAMP = Path("latest-rulesets-timestamp").read_text().strip() + + +def load_xml_files(): + xml_files = [] + for ruleset in sorted(Path("rulesets").glob("*.xml")): + tree = xml.etree.ElementTree.parse(ruleset) + root = tree.getroot() + # Rebuild the JSON + expected = { + "name": root.attrib["name"], + "target": [root[0].attrib["host"]], + "rule": [{"from": root[1].attrib["from"], "to": root[1].attrib["to"]}], + } + xml_files.append(expected) + return xml_files + + +def test_compressed_matches_xml(): + # Read the contents of the gzipped file + with gzip.open(f"default.rulesets.{LATEST_TIMESTAMP}.gz", "rb") as f: + rulesets = json.load(f) + xml_files = load_xml_files() + + assert rulesets["rulesets"] == xml_files + + +def test_generated_matches_xml(): + # Read the contents of the default file + rulesets = json.loads(Path("rulesets/default.rulesets").read_text()) + xml_files = load_xml_files() + + assert rulesets == xml_files + + +def test_unique_signature(): + """ + For every default.rulesets.*.gz file, there should be a corresponding + rulesets-signature.*.sha256 file with no extras. + """ + rulesets = [] + for path in sorted(Path(".").glob("default.rulesets.*.gz")): + rulesets.append(path.name.split(".")[2]) + + signatures = [] + for path in sorted(Path(".").glob("rulesets-signature.*.sha256")): + signatures.append(path.name.split(".")[1]) + + # Just verify it found *something* + assert LATEST_TIMESTAMP in rulesets + # Now check they're equal + assert rulesets == signatures