From dc28025e85663ad3e95005b1ee6f3fdc9b5f9717 Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Tue, 5 Nov 2024 16:14:54 +0100 Subject: [PATCH 01/12] Move tests out of source to improve dev iteration --- tests/sample_merklemap_data.jsonl | 33 ++++++++++++++++++++++++++++++ tests/sample_merklemap_data.xz | Bin 0 -> 2280 bytes 2 files changed, 33 insertions(+) create mode 100644 tests/sample_merklemap_data.jsonl create mode 100644 tests/sample_merklemap_data.xz diff --git a/tests/sample_merklemap_data.jsonl b/tests/sample_merklemap_data.jsonl new file mode 100644 index 0000000..aed9a43 --- /dev/null +++ b/tests/sample_merklemap_data.jsonl @@ -0,0 +1,33 @@ +{"hostname":"nl.dev.batch.internet.nl","results":[{"success":{"query":"nl.dev.batch.internet.nl IN A","query_timestamp":"2024-10-22T22:33:11.792584228Z","records":{"A":["62.204.66.17"],"CNAME":["dev.batch.internet.nl."]}}},{"success":{"query":"nl.dev.batch.internet.nl IN AAAA","query_timestamp":"2024-10-22T22:33:11.792588008Z","records":{"CNAME":["dev.batch.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:17"]}}},{"success":{"query":"nl.dev.batch.internet.nl IN MX","query_timestamp":"2024-10-22T22:33:11.792592598Z","records":{"MX":["10 vmx02.prolocation.nl.","10 vmx03.prolocation.net.","10 vmx01.prolocation.nl."],"CNAME":["dev.batch.internet.nl."]}}},{"success":{"query":"nl.dev.batch.internet.nl IN TXT","query_timestamp":"2024-10-22T22:33:11.792596298Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev.batch.internet.nl."]}}}]} +{"hostname":"ipv6kaartmetingen.internet.nl","results":[{"success":{"query":"ipv6kaartmetingen.internet.nl IN A","query_timestamp":"2024-10-22T22:37:21.799320664Z","records":{"A":["136.144.246.3"]}}},{"success":{"query":"ipv6kaartmetingen.internet.nl IN AAAA","query_timestamp":"2024-10-22T22:37:21.799324664Z","records":{"AAAA":["2a01:7c8:d006:35f:5054:ff:fe0a:8dd8"]}}},{"success":{"query":"ipv6kaartmetingen.internet.nl IN MX","query_timestamp":"2024-10-22T22:37:21.799328144Z","records":{"MX":["0 ."]}}},{"success":{"query":"ipv6kaartmetingen.internet.nl IN TXT","query_timestamp":"2024-10-22T22:37:21.799331544Z","records":{"TXT":["v=spf1 -all"]}}}]} +{"hostname":"matomo.internet.nl","results":[{"success":{"query":"matomo.internet.nl IN A","query_timestamp":"2024-10-23T01:20:16.052324787Z","records":{"CNAME":["emailveilig.internet.nl."],"A":["62.204.66.21"]}}},{"success":{"query":"matomo.internet.nl IN AAAA","query_timestamp":"2024-10-23T01:20:16.052327407Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:21"],"CNAME":["emailveilig.internet.nl."]}}},{"success":{"query":"matomo.internet.nl IN MX","query_timestamp":"2024-10-23T01:20:16.052329957Z","records":{"MX":["0 ."],"CNAME":["emailveilig.internet.nl."]}}},{"success":{"query":"matomo.internet.nl IN TXT","query_timestamp":"2024-10-23T01:20:16.052334647Z","records":{"TXT":["v=spf1 -all"],"CNAME":["emailveilig.internet.nl."]}}}]} +{"hostname":"helpdesk.internet.nl","results":[{"success":{"query":"helpdesk.internet.nl IN A","query_timestamp":"2024-10-23T05:50:45.075437020Z","records":{"A":["62.204.66.22"]}}},{"success":{"query":"helpdesk.internet.nl IN AAAA","query_timestamp":"2024-10-23T05:50:45.075440200Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:22"]}}},{"success":{"query":"helpdesk.internet.nl IN MX","query_timestamp":"2024-10-23T05:50:45.075445270Z","records":{"MX":["0 ."]}}},{"success":{"query":"helpdesk.internet.nl IN TXT","query_timestamp":"2024-10-23T05:50:45.075450770Z","records":{"TXT":["v=spf1 -all"]}}}]} +{"hostname":"batch.internet.nl","results":[{"success":{"query":"batch.internet.nl IN A","query_timestamp":"2024-10-23T05:53:44.985949012Z","records":{"A":["62.204.66.12"]}}},{"success":{"query":"batch.internet.nl IN AAAA","query_timestamp":"2024-10-23T05:53:44.985951812Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:12"]}}},{"success":{"query":"batch.internet.nl IN MX","query_timestamp":"2024-10-23T05:53:44.985956522Z","records":{"MX":["10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl.","10 vmx03.prolocation.net."]}}},{"success":{"query":"batch.internet.nl IN TXT","query_timestamp":"2024-10-23T05:53:44.985962642Z","records":{"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"en.conn.internet.nl","results":[{"success":{"query":"en.conn.internet.nl IN A","query_timestamp":"2024-10-23T07:56:34.556390504Z","records":{"CNAME":["www.internet.nl."],"A":["62.204.66.10"]}}},{"success":{"query":"en.conn.internet.nl IN AAAA","query_timestamp":"2024-10-23T07:56:34.556394824Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:10"],"CNAME":["www.internet.nl."]}}},{"success":{"query":"en.conn.internet.nl IN MX","query_timestamp":"2024-10-23T07:56:34.556398884Z","records":{"MX":["0 ."],"CNAME":["www.internet.nl."]}}},{"success":{"query":"en.conn.internet.nl IN TXT","query_timestamp":"2024-10-23T07:56:34.556403874Z","records":{"CNAME":["www.internet.nl."],"TXT":["v=spf1 -all"]}}}]} +{"hostname":"en.conn.dev-docker.internet.nl","results":[{"success":{"query":"en.conn.dev-docker.internet.nl IN A","query_timestamp":"2024-10-23T09:33:50.987911398Z","records":{"CNAME":["dev-docker.internet.nl."],"A":["62.204.66.35"]}}},{"success":{"query":"en.conn.dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-23T09:33:50.987915858Z","records":{"CNAME":["dev-docker.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:35"]}}},{"success":{"query":"en.conn.dev-docker.internet.nl IN MX","query_timestamp":"2024-10-23T09:33:50.987922848Z","records":{"CNAME":["dev-docker.internet.nl."],"MX":["10 vmx03.prolocation.net.","10 vmx02.prolocation.nl.","10 vmx01.prolocation.nl."]}}},{"success":{"query":"en.conn.dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-23T09:33:50.987929938Z","records":{"CNAME":["dev-docker.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"nl.dev-docker.internet.nl","results":[{"success":{"query":"nl.dev-docker.internet.nl IN A","query_timestamp":"2024-10-23T10:05:44.178092289Z","records":{"A":["62.204.66.35"],"CNAME":["dev-docker.internet.nl."]}}},{"success":{"query":"nl.dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-23T10:05:44.178096929Z","records":{"CNAME":["dev-docker.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:35"]}}},{"success":{"query":"nl.dev-docker.internet.nl IN MX","query_timestamp":"2024-10-23T10:05:44.178102419Z","records":{"CNAME":["dev-docker.internet.nl."],"MX":["10 vmx01.prolocation.nl.","10 vmx03.prolocation.net.","10 vmx02.prolocation.nl."]}}},{"success":{"query":"nl.dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-23T10:05:44.178107279Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev-docker.internet.nl."]}}}]} +{"hostname":"en.dev-docker.internet.nl","results":[{"success":{"query":"en.dev-docker.internet.nl IN A","query_timestamp":"2024-10-23T10:23:51.895551112Z","records":{"CNAME":["dev-docker.internet.nl."],"A":["62.204.66.35"]}}},{"success":{"query":"en.dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-23T10:23:51.895555983Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:35"],"CNAME":["dev-docker.internet.nl."]}}},{"success":{"query":"en.dev-docker.internet.nl IN MX","query_timestamp":"2024-10-23T10:23:51.895561852Z","records":{"MX":["10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl.","10 vmx03.prolocation.net."],"CNAME":["dev-docker.internet.nl."]}}},{"success":{"query":"en.dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-23T10:23:51.895566622Z","records":{"CNAME":["dev-docker.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"en.internet.nl","results":[{"success":{"query":"en.internet.nl IN A","query_timestamp":"2024-10-23T14:26:05.317658431Z","records":{"CNAME":["www.internet.nl."],"A":["62.204.66.10"]}}},{"success":{"query":"en.internet.nl IN AAAA","query_timestamp":"2024-10-23T14:26:05.317662311Z","records":{"CNAME":["www.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:10"]}}},{"success":{"query":"en.internet.nl IN MX","query_timestamp":"2024-10-23T14:26:05.317667201Z","records":{"MX":["0 ."],"CNAME":["www.internet.nl."]}}},{"success":{"query":"en.internet.nl IN TXT","query_timestamp":"2024-10-23T14:26:05.317671161Z","records":{"CNAME":["www.internet.nl."],"TXT":["v=spf1 -all"]}}}]} +{"hostname":"www.dev-docker.internet.nl","results":[{"success":{"query":"www.dev-docker.internet.nl IN A","query_timestamp":"2024-10-23T15:03:53.145159703Z","records":{"CNAME":["dev-docker.internet.nl."],"A":["62.204.66.35"]}}},{"success":{"query":"www.dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-23T15:03:53.145164473Z","records":{"CNAME":["dev-docker.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:35"]}}},{"success":{"query":"www.dev-docker.internet.nl IN MX","query_timestamp":"2024-10-23T15:03:53.145170523Z","records":{"CNAME":["dev-docker.internet.nl."],"MX":["10 vmx02.prolocation.nl.","10 vmx03.prolocation.net.","10 vmx01.prolocation.nl."]}}},{"success":{"query":"www.dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-23T15:03:53.145174363Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev-docker.internet.nl."]}}}]} +{"hostname":"dev.batch.internet.nl","results":[{"success":{"query":"dev.batch.internet.nl IN A","query_timestamp":"2024-10-23T15:50:15.028538521Z","records":{"A":["62.204.66.17"]}}},{"success":{"query":"dev.batch.internet.nl IN AAAA","query_timestamp":"2024-10-23T15:50:15.028541771Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:17"]}}},{"success":{"query":"dev.batch.internet.nl IN MX","query_timestamp":"2024-10-23T15:50:15.028546701Z","records":{"MX":["10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl.","10 vmx03.prolocation.net."]}}},{"success":{"query":"dev.batch.internet.nl IN TXT","query_timestamp":"2024-10-23T15:50:15.028551271Z","records":{"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"conn.internet.nl","results":[{"success":{"query":"conn.internet.nl IN A","query_timestamp":"2024-10-23T21:30:26.439068901Z","records":{"CNAME":["www.internet.nl."],"A":["62.204.66.10"]}}},{"success":{"query":"conn.internet.nl IN AAAA","query_timestamp":"2024-10-23T21:30:26.439073191Z","records":{"CNAME":["www.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:10"]}}},{"success":{"query":"conn.internet.nl IN MX","query_timestamp":"2024-10-23T21:30:26.439077411Z","records":{"CNAME":["www.internet.nl."],"MX":["0 ."]}}},{"success":{"query":"conn.internet.nl IN TXT","query_timestamp":"2024-10-23T21:30:26.439083561Z","records":{"CNAME":["www.internet.nl."],"TXT":["v=spf1 -all"]}}}]} +{"hostname":"emailveilig.internet.nl","results":[{"success":{"query":"emailveilig.internet.nl IN A","query_timestamp":"2024-10-23T23:28:24.380730112Z","records":{"A":["62.204.66.21"]}}},{"success":{"query":"emailveilig.internet.nl IN AAAA","query_timestamp":"2024-10-23T23:28:24.380732502Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:21"]}}},{"success":{"query":"emailveilig.internet.nl IN MX","query_timestamp":"2024-10-23T23:28:24.380735882Z","records":{"MX":["0 ."]}}},{"success":{"query":"emailveilig.internet.nl IN TXT","query_timestamp":"2024-10-23T23:28:24.380740652Z","records":{"TXT":["v=spf1 -all"]}}}]} +{"hostname":"www.dev-docker.batch.internet.nl","results":[{"success":{"query":"www.dev-docker.batch.internet.nl IN A","query_timestamp":"2024-10-23T23:45:02.835672913Z","records":{"A":["62.204.66.15"],"CNAME":["dev-docker.batch.internet.nl."]}}},{"success":{"query":"www.dev-docker.batch.internet.nl IN AAAA","query_timestamp":"2024-10-23T23:45:02.835676703Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:15"],"CNAME":["dev-docker.batch.internet.nl."]}}},{"success":{"query":"www.dev-docker.batch.internet.nl IN MX","query_timestamp":"2024-10-23T23:45:02.835689873Z","records":{"CNAME":["dev-docker.batch.internet.nl."],"MX":["10 vmx03.prolocation.net.","10 vmx02.prolocation.nl.","10 vmx01.prolocation.nl."]}}},{"success":{"query":"www.dev-docker.batch.internet.nl IN TXT","query_timestamp":"2024-10-23T23:45:02.835696542Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev-docker.batch.internet.nl."]}}}]} +{"hostname":"dev-docker.internet.nl","results":[{"success":{"query":"dev-docker.internet.nl IN A","query_timestamp":"2024-10-24T00:15:10.147581691Z","records":{"A":["62.204.66.35"]}}},{"success":{"query":"dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-24T00:15:10.147584881Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:35"]}}},{"success":{"query":"dev-docker.internet.nl IN MX","query_timestamp":"2024-10-24T00:15:10.147590471Z","records":{"MX":["10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl.","10 vmx03.prolocation.net."]}}},{"success":{"query":"dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-24T00:15:10.147595061Z","records":{"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"dashboard.internet.nl","results":[{"success":{"query":"dashboard.internet.nl IN A","query_timestamp":"2024-10-24T00:16:35.500205670Z","records":{"A":["149.210.144.250"]}}},{"success":{"query":"dashboard.internet.nl IN AAAA","query_timestamp":"2024-10-24T00:16:35.500209260Z","records":{"AAAA":["2a01:7c8:aab1:2c::1"]}}},{"success":{"query":"dashboard.internet.nl IN TXT","query_timestamp":"2024-10-24T00:16:35.500216210Z","records":{"TXT":["v=spf1 include:spfinc.prolocation.net ~all"]}}}]} +{"hostname":"acc.dashboard.internet.nl","results":[{"success":{"query":"acc.dashboard.internet.nl IN A","query_timestamp":"2024-10-24T00:33:24.828652559Z","records":{"A":["149.210.150.226"]}}},{"success":{"query":"acc.dashboard.internet.nl IN AAAA","query_timestamp":"2024-10-24T00:33:24.828655710Z","records":{"AAAA":["2a01:7c8:aab2:3f::1"]}}},{"success":{"query":"acc.dashboard.internet.nl IN MX","query_timestamp":"2024-10-24T00:33:24.828659470Z","records":{"MX":["0 ."]}}},{"success":{"query":"acc.dashboard.internet.nl IN TXT","query_timestamp":"2024-10-24T00:33:24.828664110Z","records":{"TXT":["v=spf1 -all"]}}}]} +{"hostname":"nl.conn.internet.nl","results":[{"success":{"query":"nl.conn.internet.nl IN A","query_timestamp":"2024-10-24T03:37:36.116653128Z","records":{"A":["62.204.66.10"],"CNAME":["www.internet.nl."]}}},{"success":{"query":"nl.conn.internet.nl IN AAAA","query_timestamp":"2024-10-24T03:37:36.116657088Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:10"],"CNAME":["www.internet.nl."]}}},{"success":{"query":"nl.conn.internet.nl IN MX","query_timestamp":"2024-10-24T03:37:36.116660268Z","records":{"MX":["0 ."],"CNAME":["www.internet.nl."]}}},{"success":{"query":"nl.conn.internet.nl IN TXT","query_timestamp":"2024-10-24T03:37:36.116665258Z","records":{"CNAME":["www.internet.nl."],"TXT":["v=spf1 -all"]}}}]} +{"hostname":"toolbox.internet.nl","results":[{"success":{"query":"toolbox.internet.nl IN A","query_timestamp":"2024-10-24T08:05:53.710558088Z","records":{"CNAME":["emailveilig.internet.nl."],"A":["62.204.66.21"]}}},{"success":{"query":"toolbox.internet.nl IN AAAA","query_timestamp":"2024-10-24T08:05:53.710562108Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:21"],"CNAME":["emailveilig.internet.nl."]}}},{"success":{"query":"toolbox.internet.nl IN MX","query_timestamp":"2024-10-24T08:05:53.710565657Z","records":{"MX":["0 ."],"CNAME":["emailveilig.internet.nl."]}}},{"success":{"query":"toolbox.internet.nl IN TXT","query_timestamp":"2024-10-24T08:05:53.710570907Z","records":{"TXT":["v=spf1 -all"],"CNAME":["emailveilig.internet.nl."]}}}]} +{"hostname":"nl.conn.dev-docker.internet.nl","results":[{"success":{"query":"nl.conn.dev-docker.internet.nl IN A","query_timestamp":"2024-10-24T13:46:04.837848047Z","records":{"CNAME":["dev-docker.internet.nl."],"A":["62.204.66.35"]}}},{"success":{"query":"nl.conn.dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-24T13:46:04.837850777Z","records":{"CNAME":["dev-docker.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:35"]}}},{"success":{"query":"nl.conn.dev-docker.internet.nl IN MX","query_timestamp":"2024-10-24T13:46:04.837855437Z","records":{"MX":["10 vmx03.prolocation.net.","10 vmx02.prolocation.nl.","10 vmx01.prolocation.nl."],"CNAME":["dev-docker.internet.nl."]}}},{"success":{"query":"nl.conn.dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-24T13:46:04.837859677Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev-docker.internet.nl."]}}}]} +{"hostname":"www.internet.nl","results":[{"success":{"query":"www.internet.nl IN A","query_timestamp":"2024-10-24T14:56:35.674670902Z","records":{"A":["62.204.66.10"]}}},{"success":{"query":"www.internet.nl IN AAAA","query_timestamp":"2024-10-24T14:56:35.674674262Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:10"]}}},{"success":{"query":"www.internet.nl IN MX","query_timestamp":"2024-10-24T14:56:35.674676892Z","records":{"MX":["0 ."]}}},{"success":{"query":"www.internet.nl IN TXT","query_timestamp":"2024-10-24T14:56:35.674681282Z","records":{"TXT":["v=spf1 -all"]}}}]} +{"hostname":"nl.batch.internet.nl","results":[{"success":{"query":"nl.batch.internet.nl IN A","query_timestamp":"2024-10-24T15:50:15.435279565Z","records":{"A":["62.204.66.12"],"CNAME":["batch.internet.nl."]}}},{"success":{"query":"nl.batch.internet.nl IN AAAA","query_timestamp":"2024-10-24T15:50:15.435284505Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:12"],"CNAME":["batch.internet.nl."]}}},{"success":{"query":"nl.batch.internet.nl IN MX","query_timestamp":"2024-10-24T15:50:15.435292495Z","records":{"MX":["10 vmx02.prolocation.nl.","10 vmx03.prolocation.net.","10 vmx01.prolocation.nl."],"CNAME":["batch.internet.nl."]}}},{"success":{"query":"nl.batch.internet.nl IN TXT","query_timestamp":"2024-10-24T15:50:15.435298595Z","records":{"CNAME":["batch.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"ipv6kaart.internet.nl","results":[{"success":{"query":"ipv6kaart.internet.nl IN A","query_timestamp":"2024-10-24T17:54:46.046966244Z","records":{"A":["93.119.13.25"]}}},{"success":{"query":"ipv6kaart.internet.nl IN AAAA","query_timestamp":"2024-10-24T17:54:46.046970194Z","records":{"AAAA":["2a01:7c8:bb09:2a:5054:ff:fe24:c0f8"]}}},{"success":{"query":"ipv6kaart.internet.nl IN MX","query_timestamp":"2024-10-24T17:54:46.046972874Z","records":{"MX":["0 ."]}}},{"success":{"query":"ipv6kaart.internet.nl IN TXT","query_timestamp":"2024-10-24T17:54:46.046977594Z","records":{"TXT":["v=spf1 -all"]}}}]} +{"hostname":"nl.internet.nl","results":[{"success":{"query":"nl.internet.nl IN A","query_timestamp":"2024-10-24T19:31:40.379357304Z","records":{"A":["62.204.66.10"],"CNAME":["www.internet.nl."]}}},{"success":{"query":"nl.internet.nl IN AAAA","query_timestamp":"2024-10-24T19:31:40.379360273Z","records":{"CNAME":["www.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:10"]}}},{"success":{"query":"nl.internet.nl IN MX","query_timestamp":"2024-10-24T19:31:40.379363084Z","records":{"CNAME":["www.internet.nl."],"MX":["0 ."]}}},{"success":{"query":"nl.internet.nl IN TXT","query_timestamp":"2024-10-24T19:31:40.379366644Z","records":{"TXT":["v=spf1 -all"],"CNAME":["www.internet.nl."]}}}]} +{"hostname":"nl.dev-docker.batch.internet.nl","results":[{"success":{"query":"nl.dev-docker.batch.internet.nl IN A","query_timestamp":"2024-10-24T19:37:22.329580341Z","records":{"CNAME":["dev-docker.batch.internet.nl."],"A":["62.204.66.15"]}}},{"success":{"query":"nl.dev-docker.batch.internet.nl IN AAAA","query_timestamp":"2024-10-24T19:37:22.329585101Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:15"],"CNAME":["dev-docker.batch.internet.nl."]}}},{"success":{"query":"nl.dev-docker.batch.internet.nl IN MX","query_timestamp":"2024-10-24T19:37:22.329594191Z","records":{"MX":["10 vmx03.prolocation.net.","10 vmx02.prolocation.nl.","10 vmx01.prolocation.nl."],"CNAME":["dev-docker.batch.internet.nl."]}}},{"success":{"query":"nl.dev-docker.batch.internet.nl IN TXT","query_timestamp":"2024-10-24T19:37:22.329600421Z","records":{"CNAME":["dev-docker.batch.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"conn.dev-docker.internet.nl","results":[{"success":{"query":"conn.dev-docker.internet.nl IN A","query_timestamp":"2024-10-24T21:32:57.567926627Z","records":{"CNAME":["dev-docker.internet.nl."],"A":["62.204.66.35"]}}},{"success":{"query":"conn.dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-24T21:32:57.567931367Z","records":{"CNAME":["dev-docker.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:35"]}}},{"success":{"query":"conn.dev-docker.internet.nl IN MX","query_timestamp":"2024-10-24T21:32:57.567938457Z","records":{"CNAME":["dev-docker.internet.nl."],"MX":["10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl.","10 vmx03.prolocation.net."]}}},{"success":{"query":"conn.dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-24T21:32:57.567942727Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev-docker.internet.nl."]}}}]} +{"hostname":"dev-docker.batch.internet.nl","results":[{"success":{"query":"dev-docker.batch.internet.nl IN A","query_timestamp":"2024-10-25T02:53:52.168466116Z","records":{"A":["62.204.66.15"]}}},{"success":{"query":"dev-docker.batch.internet.nl IN AAAA","query_timestamp":"2024-10-25T02:53:52.168468876Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:15"]}}},{"success":{"query":"dev-docker.batch.internet.nl IN MX","query_timestamp":"2024-10-25T02:53:52.168473436Z","records":{"MX":["10 vmx03.prolocation.net.","10 vmx02.prolocation.nl.","10 vmx01.prolocation.nl."]}}},{"success":{"query":"dev-docker.batch.internet.nl IN TXT","query_timestamp":"2024-10-25T02:53:52.168476006Z","records":{"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"en.dev.batch.internet.nl","results":[{"success":{"query":"en.dev.batch.internet.nl IN A","query_timestamp":"2024-10-25T03:24:21.659120598Z","records":{"CNAME":["dev.batch.internet.nl."],"A":["62.204.66.17"]}}},{"success":{"query":"en.dev.batch.internet.nl IN AAAA","query_timestamp":"2024-10-25T03:24:21.659125048Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:17"],"CNAME":["dev.batch.internet.nl."]}}},{"success":{"query":"en.dev.batch.internet.nl IN MX","query_timestamp":"2024-10-25T03:24:21.659131188Z","records":{"MX":["10 vmx02.prolocation.nl.","10 vmx03.prolocation.net.","10 vmx01.prolocation.nl."],"CNAME":["dev.batch.internet.nl."]}}},{"success":{"query":"en.dev.batch.internet.nl IN TXT","query_timestamp":"2024-10-25T03:24:21.659136068Z","records":{"CNAME":["dev.batch.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"e-mailveilig.internet.nl","results":[{"success":{"query":"e-mailveilig.internet.nl IN A","query_timestamp":"2024-10-25T04:01:58.281706753Z","records":{"CNAME":["emailveilig.internet.nl."],"A":["62.204.66.21"]}}},{"success":{"query":"e-mailveilig.internet.nl IN AAAA","query_timestamp":"2024-10-25T04:01:58.281710553Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:21"],"CNAME":["emailveilig.internet.nl."]}}},{"success":{"query":"e-mailveilig.internet.nl IN MX","query_timestamp":"2024-10-25T04:01:58.281714263Z","records":{"CNAME":["emailveilig.internet.nl."],"MX":["0 ."]}}},{"success":{"query":"e-mailveilig.internet.nl IN TXT","query_timestamp":"2024-10-25T04:01:58.281720063Z","records":{"TXT":["v=spf1 -all"],"CNAME":["emailveilig.internet.nl."]}}}]} +{"hostname":"en.dev-docker.batch.internet.nl","results":[{"success":{"query":"en.dev-docker.batch.internet.nl IN A","query_timestamp":"2024-10-25T09:56:45.958339736Z","records":{"CNAME":["dev-docker.batch.internet.nl."],"A":["62.204.66.15"]}}},{"success":{"query":"en.dev-docker.batch.internet.nl IN AAAA","query_timestamp":"2024-10-25T09:56:45.958343986Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:15"],"CNAME":["dev-docker.batch.internet.nl."]}}},{"success":{"query":"en.dev-docker.batch.internet.nl IN MX","query_timestamp":"2024-10-25T09:56:45.958349686Z","records":{"CNAME":["dev-docker.batch.internet.nl."],"MX":["10 vmx03.prolocation.net.","10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl."]}}},{"success":{"query":"en.dev-docker.batch.internet.nl IN TXT","query_timestamp":"2024-10-25T09:56:45.958354706Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev-docker.batch.internet.nl."]}}}]} +{"hostname":"en.batch.internet.nl","results":[{"success":{"query":"en.batch.internet.nl IN A","query_timestamp":"2024-10-25T10:46:16.049155460Z","records":{"A":["62.204.66.12"],"CNAME":["batch.internet.nl."]}}},{"success":{"query":"en.batch.internet.nl IN AAAA","query_timestamp":"2024-10-25T10:46:16.049161069Z","records":{"CNAME":["batch.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:12"]}}},{"success":{"query":"en.batch.internet.nl IN MX","query_timestamp":"2024-10-25T10:46:16.049168889Z","records":{"MX":["10 vmx02.prolocation.nl.","10 vmx03.prolocation.net.","10 vmx01.prolocation.nl."],"CNAME":["batch.internet.nl."]}}},{"success":{"query":"en.batch.internet.nl IN TXT","query_timestamp":"2024-10-25T10:46:16.049174329Z","records":{"CNAME":["batch.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"www.batch.internet.nl","results":[{"success":{"query":"www.batch.internet.nl IN A","query_timestamp":"2024-10-25T10:51:49.580512543Z","records":{"A":["62.204.66.12"],"CNAME":["batch.internet.nl."]}}},{"success":{"query":"www.batch.internet.nl IN AAAA","query_timestamp":"2024-10-25T10:51:49.580516143Z","records":{"CNAME":["batch.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:12"]}}},{"success":{"query":"www.batch.internet.nl IN MX","query_timestamp":"2024-10-25T10:51:49.580521363Z","records":{"CNAME":["batch.internet.nl."],"MX":["10 vmx03.prolocation.net.","10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl."]}}},{"success":{"query":"www.batch.internet.nl IN TXT","query_timestamp":"2024-10-25T10:51:49.580526103Z","records":{"CNAME":["batch.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} \ No newline at end of file diff --git a/tests/sample_merklemap_data.xz b/tests/sample_merklemap_data.xz new file mode 100644 index 0000000000000000000000000000000000000000..f0f2cd696e1e60d31524c704918ddcde8e1ae9b5 GIT binary patch literal 2280 zcmVw3Zi3c<^R#k%tmzYSw z=Nc%kgGQcGrFn76!o^r@~oA9n;Wq%$@!{_T17h>emIdLb7y;A2Cu2rR9%U;T4u=*LHp>u zb>yiOq5=gTDL0kFSA;rN`|@>etYgVTG`N{vP*T1xe%@2^d}%jdR`rFt82b=|yu zmGW^H_|&Fp4JR_Vs+TThiQU?YO_lzkl$*DmkwuM~vIG`-6I(w^AYNj(dRj=L_t*T8 zbP3ICe9E?85}A^<;DGYnP){A3sSK0l`aYGflI`EMtq8wKnPM=+%E!B zn6J>PH;I$SLBtpXtDdqsL}Lx01#2ey+VIx@DCYEVx;YIznZw05qM5OO*bR>2h;3~>dSCD;O%m8p2uSY#{383BtuMlnnMB@}24uU$!Q_an@ zhc-135dp5;cUdNN;HwhFN z!CYDVhd{lDIZSZG$56lX$H(;B;*gIOAb&Rj zy(7oqE}{WZd%aWEoEZA z>cF@u`-YA$n*+I)vSSOkQ1(VC-V+oC(2lAv3ACk7NBg~so`ePCK|hLH2lc0RGx$SOUI#` zqmv-5t00~2P~qjt8~>q#VVMMXHn^(oZ)3fb&q;8)Xff+xGt9WKrBOe1!1MVDLWaVqM5KF6^0?1JId`u%)*h!I>H zZ@qK&w?Q_UEOPn-B{q-!Zr;M@Y)eKqHI^63t~3K&7;vWATWlti+{`E=2h^&;30-<@ z@|O)ju{M!Pk)~94H0^VOMf0BsXG%j5C@XkRG-y-sD04t4p=xRxhF0bZq9a@AL|Z%DdRrPvb=@M_FFN zQQYfG;m$-r?zF;ZbJf$#{y-OF1m86pF>`Mv|8$BuZ74vT#F=~C$xNycC|`Kh-z`6g z+Nk3YEYYlfbV=uwx2EIZpg>o_ewIc%hS!C4Ej644G$c})M&Xd@&Et@x#H=8xKQR)B z7I;s|p>uq9rq|A=ZcMZo;~^UhuwZ@6j50z4u)t(?nrMH!t_hUz!n?!uZ%oJe`lTZ= z^royjvoxWM=pRc+mS8&JaULKgtYk(|c7KFhxRTuBLhDojj@RJW+REaJ1*h2Er(*AE&o^W0EZ8ggC2~ATm3)DDK`|01PqL%GT?0Lt^v|JnBeMlHQ;|9?hm za?;kB)eOr-x>R6FA}%i->eS|LBXh`Tw+o?rjiIC(8`(B zfwdr(v)w@{#fq5;U34v!PbkW8P|kNYZ($(hjExJ=sb3G%dteyWMTcx%RVqlrI$A;y z3fl^=XI7qu%Md@657kU)Rj4EPfe|A$+i7iT&(a<759?f@$1{z1Ke&Lv1EXLhCp^o+ zY+0d_xY~-REiKseBebX&=khzxc>W1pc}BGkH^~)Wpa^0l;~cIqA3iwSd=WK!H&^z9 zBHM5a2fLz|!g>cg*@sd7MMs1|%hbNSJ764v!u_1H(Svx?9f>4o*Z{r*ey=xSIL_(|#Zq)Y)i#?RZRz-)66< z7n0}849(m=Sxn`IINlEzGrsnH9Y!gdo7@W(d+nqL>X5YnoLS-KRJCyqmeQ1CIf*2V{4cPNwx zA=7014tFdxNozkR6K>iruEWS!J!j(3X<*=`wsEn)*34Z-ARi!E#_&PToVfBtMzy%& z^L8h_3Fs4iz?qyk&!33^0000s!ZZWS literal 0 HcmV?d00001 From 1f1e68975699c7c956f1b7dd7c8e35823ff215df Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Tue, 5 Nov 2024 16:59:13 +0100 Subject: [PATCH 02/12] Add CI build --- .github/workflows/build.yml | 46 +++++++++++++++++++++ .gitignore | 1 + Dockerfile | 3 ++ Makefile | 23 +++++++---- README.md | 13 ++++++ compose.yml | 18 +++++--- pyproject.toml | 5 ++- src/ctlssa/suggestions/logic/bulk_ingest.py | 1 + tests/conftest.py | 4 +- 9 files changed, 97 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..b6c89a6 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,46 @@ +name: QA and image building + +on: + pull_request: + push: + branches: + - main + +jobs: + build: + environment: Default + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + # This helps with adding version information to the build in the admin on production. CI usually works shallow. + # https://stackoverflow.com/questions/4916492/git-describe-fails-with-fatal-no-names-found-cannot-describe-anything + # this will fetch the whole repository, which is more expensive. + - run: git fetch --prune --unshallow --tags + + - name: build images + run: | + VERSION=$(git describe --tags --abbrev=0)-$(git log -1 --pretty=format:%h) + make build build_args=--build-arg=VERSION=${VERSION:-0.0.0-dev0} + + - name: lint + run: make lint + + - name: test + run: make test + + - name: Login to Docker Hub + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Push image to docker hub + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + run: | + make push_images + env: + DOCKER_USERNAME: internetstandards + secure: U7kSLhWutEWne8w0zMUN3XzWWWHnK0NKuDxrKQBtYpH5VA+ROd0js7y9uFYflqdJO/doiXE4updVM8D1aGrxfJ8Wp2Ywp5vC/6Ua+Wn+pdIi2WNBqOgCeq3+2NUMND7DpChFIA9dBiEOCnnOlRKYaz5bBhOR7trruV2E+QSCG+Scxejg+pToF/cRsQDI9zTktAr9rfWDfOjz0jfxC2NixOP8THFZhDW/vXlRWWGtmwvnawwXMdyhRYnIV/fvb2RnHeVlDXckMm+GTEcRwKZTp1Cy/lCuk8zEzxPkSvrYnAeC8Dto/KPFo9XnRNKNPygf4t7YlJ8eGvw7sp5QKH327D8+1kjspq20Aqq7IoHSHnhmYGrJSsgmoWsod6Be8PaFV/FjjfDbMvcpNspVxSPQu/h0P0POdK3+0HJBQ0j1qaYlOwlKGFyoNEN+hPOg7A/RWeV+P5wn5SxZCepL4T95uS4JIsb7SMmD1GxRTr+6ygRrKOnduBtBXFV48LALxvAwvBTka9muo924HDNsRQiI0r5gizoNPPTw+T8tzynoc7QRalih03UnEVvRxuPPz+kF+GyXQYLjVRIuPlrubnprOuW31VYEgwzAv32Ec3OZoUvx6MQvSvsignklSJgg9pucMu6apQPNkyt8GPRgUKUqQscPY1mSMhblWnaPAu7my9w= diff --git a/.gitignore b/.gitignore index 23bd691..819af31 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ suggestions_domain_25_10_2024* *export_merklemap_nl_zone_2024_20_25* .root/ *.egg-info +.build diff --git a/Dockerfile b/Dockerfile index 9fa7ab3..ba27e1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,4 +37,7 @@ ENV UWSGI_UID=nobody EXPOSE 8000 +ARG VERSION=0.0.0-dev0 +ENV SETUPTOOLS_SCM_PRETEND_VERSION=$VERSION + ENTRYPOINT [ "ctlssa" ] diff --git a/Makefile b/Makefile index 6e7d135..99a4ebe 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ SHELL = /bin/bash ctlssa = docker compose run -ti app -dev = docker compose run -ti -v ${PWD}/.root:/root --rm dev +dev = docker compose run -i -v ${PWD}/.root:/root --rm dev # run this before/after checking in/out the source all: build lint test @@ -11,28 +11,33 @@ run up: requirements docker compose up --build --remove-orphans --watch # make migration files -makemigrations: build +makemigrations: ${ctlssa} makemigrations # run development shell -dev dev-shell shell: build +dev dev-shell shell: ${dev} # fix/check linting issues -lint fix: build +lint fix: ${dev} "isort ." ${dev} "black ." # run test suite -test: build +test: ${dev} "pytest ./tests/" -# build docker container images -build: requirements - docker compose build - # updates requirements.txt files when requirements.in files have changes requirements_files = $(subst .in,.txt,$(wildcard requirements*.in)) requirements: ${requirements_files} ${requirements_files}: %.txt: %.in ${dev} pip-compile $< --output-file $@ + +# build docker container images +build: .build +.build: Dockerfile compose.yml pyproject.toml $(shell find src/) ${requirements_files} + docker compose build ${build_args} + touch $@ + +push_images: + docker compose --push app certstream diff --git a/README.md b/README.md index f61463e..c141a75 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,19 @@ Requirements for development are: - Compose - GNU make +### Checking out the code + +This repository contains submodules, so after the Git clone this submodule should be initialized. + + git clone git@github.com:internetstandards/Internet.nl-ct-log-subdomain-suggestions-api.git + git submodule update --init + +### Building the application + +Before running the application, or whenever code changes, the Docker images need to be built. For this run: + + make build + ### Running application To run the application for development run: diff --git a/compose.yml b/compose.yml index b633344..6f3f575 100644 --- a/compose.yml +++ b/compose.yml @@ -1,7 +1,15 @@ services: app: image: internetstandards/ctlssa:latest - build: . + build: &app_build + context: . + cache_from: + - internetstandards/ctlssa:latest + - type=gha + cache_to: + - type=gha + args: + BUILDKIT_INLINE_CACHE: 1 entrypoint: uwsgi ports: - 8000:8000 @@ -22,8 +30,8 @@ services: start_interval: 1s app-ingest: - build: . - image: internetstandards/ctlssa + build: *app_build + image: internetstandards/ctlssa:latest environment: CTLSSA_SECRET_KEY: '1' CTLSSA_DJANGO_DATABASE: production @@ -61,8 +69,8 @@ services: start_interval: 1s db-migrate: - build: . - image: internetstandards/ctlssa + build: *app_build + image: internetstandards/ctlssa:latest restart: on-failure environment: CTLSSA_SECRET_KEY: '1' diff --git a/pyproject.toml b/pyproject.toml index ff7ea60..e4fbafb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,8 @@ requires-python = ">=3.10" dynamic = ["dependencies", "optional-dependencies"] name = "ctlssa" readme = "README.md" -version = "1.0.0" +# overwritten during CI build using VERSION/SETUPTOOLS_SCM_PRETEND_VERSION +version = "0.0.0-dev0" [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} @@ -13,7 +14,7 @@ optional-dependencies = {dev = {file = ["requirements-dev.txt"]}, deploy = {file ctlssa = "ctlssa.manage:main" [tool.setuptools.package-data] -ctlssa = ["*.json"] +"*" = ["*.json"] [build-system] requires = [ diff --git a/src/ctlssa/suggestions/logic/bulk_ingest.py b/src/ctlssa/suggestions/logic/bulk_ingest.py index 17d5bd7..6fca653 100644 --- a/src/ctlssa/suggestions/logic/bulk_ingest.py +++ b/src/ctlssa/suggestions/logic/bulk_ingest.py @@ -2,6 +2,7 @@ import simdjson import xz + from ..logic import log from ..logic.domains import CaseOptimizedBulkInsert diff --git a/tests/conftest.py b/tests/conftest.py index cc0bbab..775aea3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ -import pytest import pathlib +import pytest + + @pytest.fixture def testdata(): return pathlib.Path(__file__).parent / "testdata" From a68e72f32578520f404c7573b8fcf8a484dabe61 Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Tue, 5 Nov 2024 20:17:23 +0100 Subject: [PATCH 03/12] Allow anonymous git clones --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index cc09f98..583e24b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "vendor/certstream"] path = vendor/certstream - url = git@github.com:CaliDog/certstream-server.git + url = https://github.com/CaliDog/certstream-server.git From 091c437ef7c6f0f379a91d7d997d744c6bc6f980 Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Tue, 5 Nov 2024 21:27:34 +0100 Subject: [PATCH 04/12] Develop/watch mode --- Dockerfile | 9 ++++++--- Makefile | 6 +++++- compose.yml | 22 ++++++++++++++++++++++ src/ctlssa/app/settings.py | 1 - src/ctlssa/suggestions/logic/ingest.py | 20 +++++++++++++++++++- 5 files changed, 52 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index ba27e1c..c807473 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,9 +10,6 @@ RUN pip install -r requirements.txt ADD README.md /src ADD pyproject.toml /src -ADD src /src/src - -RUN pip install . FROM build AS dev @@ -22,6 +19,9 @@ WORKDIR /src ADD requirements-dev.txt /src RUN pip install -r requirements-dev.txt +ADD src /src/src +RUN pip install --editable . + ENV DJANGO_SETTINGS_MODULE ctlssa.app.settings ENTRYPOINT [ "bash", "-c" ] @@ -29,6 +29,9 @@ CMD [ "bash" ] FROM build AS app +ADD src /src/src +RUN pip install --editable . + ENV DJANGO_SETTINGS_MODULE ctlssa.app.settings ENV UWSGI_MODULE ctlssa.app.wsgi ENV UWSGI_HTTP_SOCKET=:8000 diff --git a/Makefile b/Makefile index 99a4ebe..add662a 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,14 @@ SHELL = /bin/bash ctlssa = docker compose run -ti app dev = docker compose run -i -v ${PWD}/.root:/root --rm dev +db = docker compose exec -i db # run this before/after checking in/out the source all: build lint test # run the entire project run up: requirements - docker compose up --build --remove-orphans --watch + COMPOSE_PROJECT_NAME=internetnl-ctlssa docker compose up --remove-orphans --watch # make migration files makemigrations: @@ -18,6 +19,9 @@ makemigrations: dev dev-shell shell: ${dev} +dbshell: + ${db} psql --user ctlssa + # fix/check linting issues lint fix: ${dev} "isort ." diff --git a/compose.yml b/compose.yml index 6f3f575..a53a0a8 100644 --- a/compose.yml +++ b/compose.yml @@ -10,6 +10,11 @@ services: - type=gha args: BUILDKIT_INLINE_CACHE: 1 + develop: &app_develop + watch: + - path: src/ + action: sync+restart + target: /src/src/ entrypoint: uwsgi ports: - 8000:8000 @@ -19,6 +24,10 @@ services: CTLSSA_DB_ENGINE: postgresql_psycopg2 CTLSSA_DB_HOST: db CTLSSA_CERTSTREAM_SERVER_URL: ws://certstream + # uwsgi reloads on SIGTERM, so use SIGINT instead + # https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi + stop_signal: SIGINT + restart: always depends_on: db-migrate: # wait for DB migration to be completed @@ -31,6 +40,7 @@ services: app-ingest: build: *app_build + develop: *app_develop image: internetstandards/ctlssa:latest environment: CTLSSA_SECRET_KEY: '1' @@ -40,6 +50,10 @@ services: CTLSSA_CERTSTREAM_SERVER_URL: ws://certstream entrypoint: ctlssa command: ingest + restart: always + # certstream library only listens to SIGINT + # https://github.com/CaliDog/certstream-python/blob/master/certstream/core.py#L56 + stop_signal: SIGINT depends_on: db-migrate: # wait for DB migration to be completed @@ -51,6 +65,7 @@ services: image: internetstandards/certstream environment: LOG_LEVEL: error + restart: always depends_on: # no need starting certstream before we can ingest app-ingest: @@ -58,6 +73,7 @@ services: db: image: postgres:15 + restart: always environment: POSTGRES_DB: ctlssa POSTGRES_USER: ctlssa @@ -67,9 +83,12 @@ services: interval: 5m start_period: 30s start_interval: 1s + volumes: + - postgres:/var/lib/postgresql/data db-migrate: build: *app_build + develop: *app_develop image: internetstandards/ctlssa:latest restart: on-failure environment: @@ -88,3 +107,6 @@ services: target: dev volumes: - .:/src + +volumes: + postgres: diff --git a/src/ctlssa/app/settings.py b/src/ctlssa/app/settings.py index 04488b7..ec245b4 100644 --- a/src/ctlssa/app/settings.py +++ b/src/ctlssa/app/settings.py @@ -17,7 +17,6 @@ # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent - # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ diff --git a/src/ctlssa/suggestions/logic/ingest.py b/src/ctlssa/suggestions/logic/ingest.py index 71c40ec..3114db3 100644 --- a/src/ctlssa/suggestions/logic/ingest.py +++ b/src/ctlssa/suggestions/logic/ingest.py @@ -1,3 +1,5 @@ +import time + import certstream from django.conf import settings @@ -18,6 +20,22 @@ def certstream_callback(message, context): add_domains(all_domains) +def listen_for_events( + message_callback, url, skip_heartbeats=True, setup_logger=True, on_open=None, on_error=None, **kwargs +): + try: + while True: + c = certstream.core.CertStreamClient( + message_callback, url, skip_heartbeats=skip_heartbeats, on_open=on_open, on_error=on_error + ) + c.run_forever(ping_interval=15, **kwargs) + time.sleep(5) + except KeyboardInterrupt: + log.info("Kill command received, exiting!!") + except: + log.exception("unhandled exception") + + def ingest_certstream(): log.info("Starting certstream ingestion from %s", settings.CERTSTREAM_SERVER_URL) - certstream.listen_for_events(certstream_callback, url=settings.CERTSTREAM_SERVER_URL) + listen_for_events(certstream_callback, url=settings.CERTSTREAM_SERVER_URL) From b8212ae5aae7934dd8bb6573d73882b74ead461d Mon Sep 17 00:00:00 2001 From: stitch1 Date: Wed, 6 Nov 2024 10:16:34 +0100 Subject: [PATCH 05/12] add historical retrieval, improve readme --- Makefile | 6 + README.md | 13 +- src/ctlssa/suggestions/fixtures/testdata.json | 534 +++++++++++++++++- src/ctlssa/suggestions/logic/bulk_ingest.py | 2 + src/ctlssa/suggestions/logic/suggest.py | 54 +- src/ctlssa/suggestions/views.py | 1 + tests/test_suggest.py | 44 +- 7 files changed, 617 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index add662a..79c8539 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,12 @@ lint fix: test: ${dev} "pytest ./tests/" +testcase: + # support extra verbosity in testcases so differences can be copy-pasted when needed and to see exactly + # what is happening. + # make testcase case=filter condition + ${dev} "pytest -vvv -k ${case} ./tests/" + # updates requirements.txt files when requirements.in files have changes requirements_files = $(subst .in,.txt,$(wildcard requirements*.in)) requirements: ${requirements_files} diff --git a/README.md b/README.md index c141a75..cf793bc 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,13 @@ fixture might be present. This contains 5.991.724 records. -> Git refused this, ## What are the limits of this tool The limits have not yet been discovered and no optimizations have been performed yet, aside from a few proactive -database indexes. It is expected to being able to store about a years worth of data from the .nl zone. This means -about 5 million domains with an estimated 50 million subdomains, each which will have a new certificate every 90 days. -In total about 200 million records per year. This is the same in most EU countries. There is no expectation that this -tool will work quickly on the combined com/net/org zones. Although some partitioning and smarter inserting might just -do the trick. For the Netherlands the total number of certificate renewals seems to be much lower for subdomains, -between 0.5 to 2 per second. +database indexes. It is expected to being able to store about a years worth of data from the .nl zone. The preload +from merklemap will load up 6 million subdomains (which is much lower than the expected 50 million). + +For the Netherlands the total number of certificate renewals seems to be much lower for subdomains, +between 0.5 to 2 per second. Each subdomain which will have a new certificate every 90 days. This is the same in +most EU countries. There is no expectation that this tool will work quickly on the combined com/net/org zones. +Although some partitioning and smarter inserting might just do the trick. The goal is to being able to run this on medium sized virtual machines with just a few cores and a few gigabytes of ram. That should be enough for the Netherlands and most EU countries. We've not tried to see if this solution is 'web diff --git a/src/ctlssa/suggestions/fixtures/testdata.json b/src/ctlssa/suggestions/fixtures/testdata.json index 2cedfbb..df26863 100644 --- a/src/ctlssa/suggestions/fixtures/testdata.json +++ b/src/ctlssa/suggestions/fixtures/testdata.json @@ -1 +1,533 @@ -[{"model": "suggestions.domain", "pk": 30116, "fields": {"subdomain": "plan", "domain": "roughmen", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30117, "fields": {"subdomain": "planning", "domain": "roughmen", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30119, "fields": {"subdomain": "www", "domain": "praktijkbosveld", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30121, "fields": {"subdomain": "www", "domain": "bureauhofmeijer", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30123, "fields": {"subdomain": "www", "domain": "odeaanlent", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30125, "fields": {"subdomain": "www", "domain": "odeaanlent", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30127, "fields": {"subdomain": "www", "domain": "qrisicoadviseurs", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30128, "fields": {"subdomain": "r.fin-acc", "domain": "newomij", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30129, "fields": {"subdomain": "lime", "domain": "wwwdeltalloyd", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30131, "fields": {"subdomain": "www", "domain": "byaniek", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30133, "fields": {"subdomain": "www", "domain": "byaniek", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30135, "fields": {"subdomain": "www", "domain": "erciyeshalal", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30136, "fields": {"subdomain": "www", "domain": "yaabba", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30138, "fields": {"subdomain": "www", "domain": "scentful", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30139, "fields": {"subdomain": "pod", "domain": "hanos", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30142, "fields": {"subdomain": "www", "domain": "erciyeshalal", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30144, "fields": {"subdomain": "pub.math", "domain": "leidenuniv", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30145, "fields": {"subdomain": "www.math", "domain": "leidenuniv", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30146, "fields": {"subdomain": "pub.math", "domain": "leidenuniv", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30147, "fields": {"subdomain": "www.math", "domain": "leidenuniv", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30148, "fields": {"subdomain": "qoroyo", "domain": "etcbc", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30287, "fields": {"subdomain": "www", "domain": "bomenriddershardenberg", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30289, "fields": {"subdomain": "www", "domain": "crossfitharderwijk", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30290, "fields": {"subdomain": "dev", "domain": "afotos", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30291, "fields": {"subdomain": "dev", "domain": "afotos", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30293, "fields": {"subdomain": "www", "domain": "eigenbenen", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30295, "fields": {"subdomain": "www", "domain": "eigenbenen", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30297, "fields": {"subdomain": "www", "domain": "eigenbenen", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30298, "fields": {"subdomain": "autodiscover", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30299, "fields": {"subdomain": "cpanel", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30300, "fields": {"subdomain": "cpcalendars", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30301, "fields": {"subdomain": "cpcontacts", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30303, "fields": {"subdomain": "ipv6", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30304, "fields": {"subdomain": "mail", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30305, "fields": {"subdomain": "webdisk", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30306, "fields": {"subdomain": "webmail", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30307, "fields": {"subdomain": "www", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30308, "fields": {"subdomain": "autodiscover", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30309, "fields": {"subdomain": "cpanel", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30310, "fields": {"subdomain": "cpcalendars", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30311, "fields": {"subdomain": "cpcontacts", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30313, "fields": {"subdomain": "ipv6", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30314, "fields": {"subdomain": "mail", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30315, "fields": {"subdomain": "webdisk", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30316, "fields": {"subdomain": "webmail", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30317, "fields": {"subdomain": "www", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30318, "fields": {"subdomain": "autodiscover", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30319, "fields": {"subdomain": "cpanel", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30320, "fields": {"subdomain": "cpcalendars", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30321, "fields": {"subdomain": "cpcontacts", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30323, "fields": {"subdomain": "ipv6", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30324, "fields": {"subdomain": "mail", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30325, "fields": {"subdomain": "webdisk", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30326, "fields": {"subdomain": "webmail", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30327, "fields": {"subdomain": "www", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30329, "fields": {"subdomain": "www", "domain": "henkbexkens", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30331, "fields": {"subdomain": "www", "domain": "henkbexkens", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30333, "fields": {"subdomain": "www", "domain": "kiefttweewielers", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30335, "fields": {"subdomain": "www", "domain": "kiefttweewielers", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30336, "fields": {"subdomain": "sacraea-saturna", "domain": "mattdrums", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30337, "fields": {"subdomain": "sacraea-saturna", "domain": "mattdrums", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30339, "fields": {"subdomain": "www", "domain": "jelgerhuis", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30341, "fields": {"subdomain": "www", "domain": "jelgerhuis", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30343, "fields": {"subdomain": "www", "domain": "doppinie", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30345, "fields": {"subdomain": "www", "domain": "doppinie", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30347, "fields": {"subdomain": "www", "domain": "doppinie", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30348, "fields": {"subdomain": "www", "domain": "waterspektakel", "suffix": "nl", "last_seen": "2024-10-28"}}] \ No newline at end of file +[ + { + "model": "suggestions.domain", + "fields": { + "subdomain": "plan", + "domain": "test", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "planning", + "domain": "test", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "nos", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "nu", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "dubbelresultaat", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "dubbelresultaat", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "autodiscover", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpanel", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpcalendars", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpcontacts", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "ipv6", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "mail", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "webdisk", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "webmail", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "autodiscover", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpanel", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpcalendars", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpcontacts", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "ipv6", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "mail", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "webdisk", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "webmail", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "autodiscover", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpanel", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpcalendars", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpcontacts", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "ipv6", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "mail", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "webdisk", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "webmail", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "example", + "suffix": "com", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "example", + "suffix": "politie", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "januari", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-01-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "februari", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-02-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "maart", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-03-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "april", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-04-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "mei", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-05-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "juni", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-06-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "juli", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-07-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "augustus", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-08-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "september", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-09-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "oktober", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-10-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "november", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-11-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "december", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-12-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "januari", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-01-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "februari", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-02-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "maart", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-03-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "april", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-04-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "mei", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-05-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "juni", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-06-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "juli", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-07-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "augustus", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-08-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "september", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-09-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "oktober", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-10-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "november", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-11-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "december", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-12-01" + } + } +] \ No newline at end of file diff --git a/src/ctlssa/suggestions/logic/bulk_ingest.py b/src/ctlssa/suggestions/logic/bulk_ingest.py index 6fca653..46d32c0 100644 --- a/src/ctlssa/suggestions/logic/bulk_ingest.py +++ b/src/ctlssa/suggestions/logic/bulk_ingest.py @@ -68,6 +68,8 @@ def ingest_merklemap(file: str = "merklemap_data.jsonl"): with my_open(file) as f: for line in f: data = parser.parse(line) + # If you load up the django fixture here instead of the merklemap file, you'll get an integer + # error here. But no try-except as that takes extra time to process. hostname = data["hostname"] # performance trick from the simdjson manual del data diff --git a/src/ctlssa/suggestions/logic/suggest.py b/src/ctlssa/suggestions/logic/suggest.py index ddb7e68..57cd972 100644 --- a/src/ctlssa/suggestions/logic/suggest.py +++ b/src/ctlssa/suggestions/logic/suggest.py @@ -1,27 +1,45 @@ -from datetime import datetime, timedelta, timezone +import contextlib +from datetime import datetime, timedelta, timezone, date from ..models import Domain -def suggest_subdomains(domain: str, suffix: str = "nl", period_in_days: int = 365): +def suggest_subdomains(domain: str, suffix: str = "nl", period_in_days: int = 365, max_date: str = ""): if not domain or not suffix or not period_in_days: return [] - - some_time_ago = datetime.now(timezone.utc) - timedelta(days=period_in_days) - - # These are not distinct, as ct logs contain certificate requests and requests can be processed multiple times + # Results are not distinct, as ct logs contain certificate requests and requests can be processed multiple times # per day. But even if a request is performed a few times per day for a domain the below logic will be # more than fast enough to reduce this set to unique values. A 'distinct' query is not needed and if performance # issues arise, distinct might be worth considering. - return sorted( - set( - Domain.objects.filter( - # not performing a last_seen__lte=now() as that is yet another thing for the database to process, - # which makes querying slower. There is no strict need for windowing over time, only a need for showing - # accurate results today. - domain=domain, - suffix=suffix, - last_seen__gte=some_time_ago, - ).values_list("subdomain", flat=True) - ) - ) + + # The reason for using two distinct queries is to see if there are performance differences between the two for + # current and historical data. The assumption is that less filtering results in better performance. We'll see + # on the long run how this pans out. + + # silently fall back if the date value is not according to specification + with contextlib.suppress(ValueError): + max_date = date.fromisoformat(max_date) + + if max_date: + some_time_ago = max_date - timedelta(days=period_in_days) + query = Domain.objects.filter( + # not performing a last_seen__lte=now() as that is yet another thing for the database to process, + # which makes querying slower. There is no strict need for windowing over time, only a need for showing + # accurate results today. + domain=domain, + suffix=suffix, + last_seen__gte=some_time_ago, + last_seen__lte=max_date, + ).values_list("subdomain", flat=True) + else: + some_time_ago = datetime.now(timezone.utc) - timedelta(days=period_in_days) + query = Domain.objects.filter( + # not performing a last_seen__lte=now() as that is yet another thing for the database to process, + # which makes querying slower. There is no strict need for windowing over time, only a need for showing + # accurate results today. + domain=domain, + suffix=suffix, + last_seen__gte=some_time_ago, + ).values_list("subdomain", flat=True) + + return sorted(set(query)) diff --git a/src/ctlssa/suggestions/views.py b/src/ctlssa/suggestions/views.py index 72b126a..97e5c0b 100644 --- a/src/ctlssa/suggestions/views.py +++ b/src/ctlssa/suggestions/views.py @@ -9,6 +9,7 @@ def suggest(request): request.GET.get("domain", None), request.GET.get("suffix", "nl"), request.GET.get("period_in_days", 365), + request.GET.get("max_date", None), ), safe=False, ) diff --git a/tests/test_suggest.py b/tests/test_suggest.py index 4800b61..e96a86c 100644 --- a/tests/test_suggest.py +++ b/tests/test_suggest.py @@ -29,25 +29,45 @@ def test_suggest_subdomains(db): "www", ] + assert suggest_subdomains("example", "com", 365) == ["www"] + assert suggest_subdomains("dubbelresultaat", "nl", 365) == ["www"] + # incomplete call of the method yields no data: assert suggest_subdomains("example", "nl", 0) == [] assert suggest_subdomains("example", "", 365) == [] assert suggest_subdomains("", "nl", 365) == [] - # this will yield the same data as above as we don't support time travel in search - with freeze_time("2024-1-30 12:00:00"): - assert suggest_subdomains("example", "nl", 10) == [ - "autodiscover", - "cpanel", - "cpcalendars", - "cpcontacts", - "ipv6", - "mail", - "webdisk", - "webmail", - "www", + with freeze_time("2024-12-30 12:00:00"): + # This will result in no data, as in 10 days no domains from 2024 have been seen after december first, + # so looking 10 days in the past results in no data. + assert suggest_subdomains("2024", "nl", 10) == [] + + # broaden the time frame to get results. + assert suggest_subdomains("2024", "nl", 30) == ["december"] + assert suggest_subdomains("2024", "nl", 65) == ["december", "november"] + + with freeze_time("2023-12-30 12:00:00"): + # Now you will get everything from 2024, because in real life you cannot go back in time :) + assert suggest_subdomains("2024", "nl", 365) == [ + "april", + "augustus", + "december", + "februari", + "januari", + "juli", + "juni", + "maart", + "mei", + "november", + "oktober", + "september", ] + # testing a time-window + with freeze_time("2024-12-30 12:00:00"): + assert suggest_subdomains("2023", "nl", 10, "2023-10-01") == ["oktober"] + assert suggest_subdomains("2023", "nl", 65, "2023-10-01") == ["augustus", "oktober", "september"] + # data is not present yet in this period, so don't suggest it. with freeze_time("2025-12-30 12:00:00"): assert suggest_subdomains("example", "nl", 10) == [] From 80ee7cf3385e4405f3d72fd525550362c36de0cc Mon Sep 17 00:00:00 2001 From: stitch1 Date: Wed, 6 Nov 2024 10:20:58 +0100 Subject: [PATCH 06/12] add dbshell documentation to readme --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index cf793bc..2879cad 100644 --- a/README.md +++ b/README.md @@ -170,3 +170,12 @@ To open a shell with all dependencies and development tools installed run: After changing requirements in any of the `.in` files update the `.txt` files using: make requirements + +### Database shell (postgresql) + + make dbshell + +Or alternatively you can: + + docker ps + docker exec -ti internetnl-ctlssa-db-1 psql --user ctlssa From 823a948560ca9e3f83be4494bf72cef06092a5b8 Mon Sep 17 00:00:00 2001 From: stitch1 Date: Wed, 6 Nov 2024 10:46:22 +0100 Subject: [PATCH 07/12] fix default max_date bug, change standard port to 8001 --- Dockerfile | 5 +++-- README.md | 6 +++--- compose.yml | 4 ++-- src/ctlssa/suggestions/logic/suggest.py | 3 +++ 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index c807473..2a71dbc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,11 +34,12 @@ RUN pip install --editable . ENV DJANGO_SETTINGS_MODULE ctlssa.app.settings ENV UWSGI_MODULE ctlssa.app.wsgi -ENV UWSGI_HTTP_SOCKET=:8000 +ENV UWSGI_HTTP_SOCKET=:8001 ENV UWSGI_MASTER=1 ENV UWSGI_UID=nobody +ENV DJANGO_PORT=8001 -EXPOSE 8000 +EXPOSE 8001 ARG VERSION=0.0.0-dev0 ENV SETUPTOOLS_SCM_PRETEND_VERSION=$VERSION diff --git a/README.md b/README.md index 2879cad..5721456 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,8 @@ This command should run forever. In case your certstream server is down it will The webserver can be started with the command: ```python manage.py runserver``` -When you visit the web interface at http://localhost:8000/ you will see a blank JSON response. Use the following -parameters to retrieve data: `http://localhost:8000/?domain=example&suffix=nl&period=365` +When you visit the web interface at http://localhost:8001/ you will see a blank JSON response. Use the following +parameters to retrieve data: `http://localhost:8001/?domain=example&suffix=nl&period=365` ## Further configuration options @@ -143,7 +143,7 @@ To run the application for development run: make run -A web interface will be available at `http://localhost:8000`. +A web interface will be available at `http://localhost:8001`. ### Linting diff --git a/compose.yml b/compose.yml index a53a0a8..8bb373f 100644 --- a/compose.yml +++ b/compose.yml @@ -17,7 +17,7 @@ services: target: /src/src/ entrypoint: uwsgi ports: - - 8000:8000 + - 8001:8001 environment: CTLSSA_SECRET_KEY: '1' CTLSSA_DJANGO_DATABASE: production @@ -33,7 +33,7 @@ services: # wait for DB migration to be completed condition: service_completed_successfully healthcheck: - test: curl --silent --fail http://127.0.0.1:8000 + test: curl --silent --fail http://127.0.0.1:8001 interval: 5m start_period: 30s start_interval: 1s diff --git a/src/ctlssa/suggestions/logic/suggest.py b/src/ctlssa/suggestions/logic/suggest.py index 57cd972..c007cae 100644 --- a/src/ctlssa/suggestions/logic/suggest.py +++ b/src/ctlssa/suggestions/logic/suggest.py @@ -18,6 +18,9 @@ def suggest_subdomains(domain: str, suffix: str = "nl", period_in_days: int = 36 # silently fall back if the date value is not according to specification with contextlib.suppress(ValueError): + # suppress "isoformat must be a string" + if not max_date: + max_date = "" max_date = date.fromisoformat(max_date) if max_date: From c3907cdaf5e2d48e9c8813c208fa0e8ed55651e2 Mon Sep 17 00:00:00 2001 From: stitch1 Date: Wed, 6 Nov 2024 10:46:48 +0100 Subject: [PATCH 08/12] fix lint --- src/ctlssa/suggestions/logic/suggest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ctlssa/suggestions/logic/suggest.py b/src/ctlssa/suggestions/logic/suggest.py index c007cae..56cb3f7 100644 --- a/src/ctlssa/suggestions/logic/suggest.py +++ b/src/ctlssa/suggestions/logic/suggest.py @@ -1,5 +1,5 @@ import contextlib -from datetime import datetime, timedelta, timezone, date +from datetime import date, datetime, timedelta, timezone from ..models import Domain From eec43edc7286e17562b01e1f3cb5739b2a14e94a Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Tue, 5 Nov 2024 21:27:34 +0100 Subject: [PATCH 09/12] Develop/watch mode --- .github/workflows/build.yml | 3 ++ Makefile | 24 +++++++++----- README.md | 49 +++++++++++++++++++++------- compose.yml | 9 ++++++ requirements-dev.in | 1 + requirements-dev.txt | 11 ++++++- requirements.txt | 64 ++++++++++--------------------------- 7 files changed, 92 insertions(+), 69 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b6c89a6..c9adf60 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,9 @@ jobs: VERSION=$(git describe --tags --abbrev=0)-$(git log -1 --pretty=format:%h) make build build_args=--build-arg=VERSION=${VERSION:-0.0.0-dev0} + - name: check + run: make check + - name: lint run: make lint diff --git a/Makefile b/Makefile index 79c8539..53c3a36 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,13 @@ SHELL = /bin/bash -ctlssa = docker compose run -ti app -dev = docker compose run -i -v ${PWD}/.root:/root --rm dev +dev = docker compose --profile=dev run -i --rm dev db = docker compose exec -i db # run this before/after checking in/out the source all: build lint test # run the entire project -run up: requirements +run up: COMPOSE_PROJECT_NAME=internetnl-ctlssa docker compose up --remove-orphans --watch # make migration files @@ -29,7 +28,17 @@ lint fix: # run test suite test: - ${dev} "pytest ./tests/" + ${dev} "pytest --verbose ." + +test-watch: + ${dev} "ptw --clear . -- --verbose" + +check: requirements + @if [ ! -z "$(shell git status --porcelain requirements*.txt)" ];then \ + echo "Requirements .in files have not all been compiled into .txt files and commited to Git!"; \ + git status --porcelain requirements*.txt; \ + exit 1; \ + fi testcase: # support extra verbosity in testcases so differences can be copy-pasted when needed and to see exactly @@ -41,13 +50,12 @@ testcase: requirements_files = $(subst .in,.txt,$(wildcard requirements*.in)) requirements: ${requirements_files} ${requirements_files}: %.txt: %.in - ${dev} pip-compile $< --output-file $@ + ${dev} "pip-compile $< --output-file $@" # build docker container images -build: .build -.build: Dockerfile compose.yml pyproject.toml $(shell find src/) ${requirements_files} +.PHONY: build +build: docker compose build ${build_args} - touch $@ push_images: docker compose --push app certstream diff --git a/README.md b/README.md index 5721456..0f5db59 100644 --- a/README.md +++ b/README.md @@ -137,19 +137,31 @@ Before running the application, or whenever code changes, the Docker images need make build +Running the build step before every command is recommended to ensure the images are up to date, that's why it's included in all the command below. It can be omitted if you know you don't need it. + ### Running application -To run the application for development run: +To run the application for development use: + + make build run - make run +This will start a full development stack with: + + - uWSGI application + - ingest + - certstream server + - postgreSQL server + - database migration A web interface will be available at `http://localhost:8001`. +Compose will watch for file changes and restart the required services accordingly. + ### Linting Run these commands before checking in. These should all pass without error. - make lint + make build lint *notice*: this command autofixes trivial issues in formatting and updates the source files @@ -157,25 +169,38 @@ Run these commands before checking in. These should all pass without error. To run the test suite use: - make test + make build test + +To rerun tests every time a file changes use: + + make build test-watch ### Development shell -To open a shell with all dependencies and development tools installed run: +To open a shell with all dependencies and development tools installed, first bring the project up, then run: + + make build dev + +From here you can run the `ctlssa` command to perform Django `./manage.py` functions like: - make dev + ctlssa makemigrations + + ctlssa shell + + ctlssa loadfixtures testdata + + ctlssa bulk_ingest --file data.json + +For all commands run: + + ctlssa --help ### Dependency management After changing requirements in any of the `.in` files update the `.txt` files using: - make requirements + make build requirements ### Database shell (postgresql) make dbshell - -Or alternatively you can: - - docker ps - docker exec -ti internetnl-ctlssa-db-1 psql --user ctlssa diff --git a/compose.yml b/compose.yml index 8bb373f..3a377e4 100644 --- a/compose.yml +++ b/compose.yml @@ -32,6 +32,8 @@ services: db-migrate: # wait for DB migration to be completed condition: service_completed_successfully + app-ingest: + condition: service_started healthcheck: test: curl --silent --fail http://127.0.0.1:8001 interval: 5m @@ -107,6 +109,13 @@ services: target: dev volumes: - .:/src + - ./.root:/root + depends_on: + app: + condition: service_healthy + + profiles: + - dev volumes: postgres: diff --git a/requirements-dev.in b/requirements-dev.in index c23de7d..3d5a7fc 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -8,3 +8,4 @@ coverage django-coverage pytest-cov pip-tools +pytest-watch diff --git a/requirements-dev.txt b/requirements-dev.txt index a4b84e1..82317fe 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile requirements-dev.in +# pip-compile --output-file=requirements-dev.txt requirements-dev.in # black==24.10.0 # via -r requirements-dev.in @@ -12,12 +12,16 @@ click==8.1.7 # via # black # pip-tools +colorama==0.4.6 + # via pytest-watch coverage[toml]==7.6.4 # via # -r requirements-dev.in # pytest-cov django-coverage==1.2.4 # via -r requirements-dev.in +docopt==0.6.2 + # via pytest-watch freezegun==1.5.1 # via -r requirements-dev.in iniconfig==2.0.0 @@ -47,14 +51,19 @@ pytest==8.3.3 # via # pytest-cov # pytest-django + # pytest-watch pytest-cov==5.0.0 # via -r requirements-dev.in pytest-django==4.9.0 # via -r requirements-dev.in +pytest-watch==4.2.0 + # via -r requirements-dev.in python-dateutil==2.9.0.post0 # via freezegun six==1.16.0 # via python-dateutil +watchdog==6.0.0 + # via pytest-watch wheel==0.44.0 # via pip-tools diff --git a/requirements.txt b/requirements.txt index 4159105..3e9c832 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,76 +5,44 @@ # pip-compile # asgiref==3.8.1 - # via - # -c requirements.txt - # django + # via django certifi==2024.8.30 - # via - # -c requirements.txt - # requests + # via requests certstream==1.12 - # via - # -c requirements.txt - # -r requirements.in + # via -r requirements.in charset-normalizer==3.4.0 - # via - # -c requirements.txt - # requests + # via requests colorlog==6.8.2 - # via - # -c requirements.txt - # -r requirements.in + # via -r requirements.in django==4.2.16 - # via - # -c requirements.txt - # -r requirements.in + # via -r requirements.in filelock==3.16.1 - # via - # -c requirements.txt - # tldextract + # via tldextract idna==3.10 # via - # -c requirements.txt # requests # tldextract +psycopg==3.2.3 + # via -r requirements.in pysimdjson==6.0.2 # via -r requirements.in python-xz==0.5.0 # via -r requirements.in -psycopg==3.2.3 - # via - # -c requirements.txt - # -r requirements.in requests==2.32.3 # via - # -c requirements.txt # requests-file # tldextract requests-file==2.1.0 - # via - # -c requirements.txt - # tldextract + # via tldextract sqlparse==0.5.1 - # via - # -c requirements.txt - # django + # via django termcolor==2.5.0 - # via - # -c requirements.txt - # certstream + # via certstream tldextract==5.1.2 - # via - # -c requirements.txt - # -r requirements.in + # via -r requirements.in typing-extensions==4.12.2 - # via - # -c requirements.txt - # psycopg + # via psycopg urllib3==2.2.3 - # via - # -c requirements.txt - # requests + # via requests websocket-client==1.8.0 - # via - # -c requirements.txt - # certstream + # via certstream From 69adaf1ff3005c3bcd817a52db3aa60544828b2b Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Wed, 6 Nov 2024 11:04:52 +0100 Subject: [PATCH 10/12] Add cleanup --- Makefile | 10 +++++++++- README.md | 6 ++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 53c3a36..23fe1a9 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,14 @@ SHELL = /bin/bash dev = docker compose --profile=dev run -i --rm dev db = docker compose exec -i db +project_name = internetnl-ctlssa + # run this before/after checking in/out the source all: build lint test # run the entire project run up: - COMPOSE_PROJECT_NAME=internetnl-ctlssa docker compose up --remove-orphans --watch + COMPOSE_PROJECT_NAME=${project_name} docker compose up --remove-orphans --watch # make migration files makemigrations: @@ -59,3 +61,9 @@ build: push_images: docker compose --push app certstream + +# remove all runtime state and cache from the project +mrproper: + docker compose rm --volumes --force --stop + docker system prune --filter label=com.docker.compose.project=${project_name} --all --force --volumes + rm -fr build/ src/*.egg-info diff --git a/README.md b/README.md index 0f5db59..024e9a9 100644 --- a/README.md +++ b/README.md @@ -204,3 +204,9 @@ After changing requirements in any of the `.in` files update the `.txt` files us ### Database shell (postgresql) make dbshell + +### Reset + +If nothing else works, try resetting the project state by deleting all runtime state and caches using: + + make mrproper From cf0e5c9e130e1a2bbb2189c707d102ac7fa30a45 Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Wed, 6 Nov 2024 11:10:47 +0100 Subject: [PATCH 11/12] Fix project name for commands --- Makefile | 12 +++++++----- compose.yml | 5 +++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 23fe1a9..7992727 100644 --- a/Makefile +++ b/Makefile @@ -3,18 +3,20 @@ SHELL = /bin/bash dev = docker compose --profile=dev run -i --rm dev db = docker compose exec -i db -project_name = internetnl-ctlssa - # run this before/after checking in/out the source all: build lint test # run the entire project run up: - COMPOSE_PROJECT_NAME=${project_name} docker compose up --remove-orphans --watch + docker compose up --remove-orphans --watch # make migration files makemigrations: - ${ctlssa} makemigrations + ${dev} "ctlssa makemigrations" + +# load testdatta fixture +testdata: + ${dev} "ctlssa loaddata testdata" # run development shell dev dev-shell shell: @@ -65,5 +67,5 @@ push_images: # remove all runtime state and cache from the project mrproper: docker compose rm --volumes --force --stop - docker system prune --filter label=com.docker.compose.project=${project_name} --all --force --volumes + docker system prune --filter label=com.docker.compose.project="$(shell docker compose config --format yaml | sed -nE 's/^name: (.*)/\1/p')" --all --force --volumes rm -fr build/ src/*.egg-info diff --git a/compose.yml b/compose.yml index 3a377e4..f1c26ed 100644 --- a/compose.yml +++ b/compose.yml @@ -107,6 +107,11 @@ services: dev: build: target: dev + environment: + CTLSSA_SECRET_KEY: '1' + CTLSSA_DJANGO_DATABASE: production + CTLSSA_DB_ENGINE: postgresql_psycopg2 + CTLSSA_DB_HOST: db volumes: - .:/src - ./.root:/root From 0368d8b7bb6181de9e54f109eaf44b9c88f7c387 Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Wed, 6 Nov 2024 15:01:02 +0100 Subject: [PATCH 12/12] Reduce certstream CPU load --- compose.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/compose.yml b/compose.yml index f1c26ed..8d81a02 100644 --- a/compose.yml +++ b/compose.yml @@ -29,6 +29,8 @@ services: stop_signal: SIGINT restart: always depends_on: + db: + condition: service_healthy db-migrate: # wait for DB migration to be completed condition: service_completed_successfully @@ -56,10 +58,15 @@ services: # certstream library only listens to SIGINT # https://github.com/CaliDog/certstream-python/blob/master/certstream/core.py#L56 stop_signal: SIGINT + stop_grace_period: 3s depends_on: + db: + condition: service_healthy db-migrate: # wait for DB migration to be completed condition: service_completed_successfully + certstream: + condition: service_healthy certstream: build: @@ -68,10 +75,11 @@ services: environment: LOG_LEVEL: error restart: always - depends_on: - # no need starting certstream before we can ingest - app-ingest: - condition: service_started + healthcheck: + test: wget -q 127.0.0.1:4000 -O /dev/null + interval: 5m + start_period: 30s + start_interval: 1s db: image: postgres:15 @@ -85,6 +93,7 @@ services: interval: 5m start_period: 30s start_interval: 1s + cpu_percent: ${CERTSTREAM_CPU_PERCENT:-10} volumes: - postgres:/var/lib/postgresql/data