Add stats

sql/stats.sql

@@ -0,0 +1,82 @@
+SET
+memory_limit = '2GB';
+SET
+threads = 4;
+
+
+COPY
+(
+select count(*) as "total_files",
+       approx_count_distinct(hash)::bigint as "unique_files", sum(size)::bigint                   as "total_size", sum(lines)::bigint                  as "total_lines",
+from 'data/*.parquet' ) TO 'stats/general_stats.json';
+
+COPY
+(
+select regexp_extract(path, '\.[0-9a-z]+$') as extension,
+       count()                              as total,
+       sum(lines)::bigint                           as lines, sum(size) ::bigint as size,
+from 'data/*.parquet'
+group by extension
+order by total DESC
+    limit 10
+    ) TO 'stats/top_extensions.json';
+
+COPY
+(
+select regexp_extract(path, '\.[0-9a-z]+$') as extension,
+       count()                              as total,
+       sum(size) ::bigint as size,
+from 'data/*.parquet'
+where skip_reason = 'binary'
+group by extension
+order by total DESC
+    limit 10
+    ) TO 'stats/top_binary_extensions.json' (ARRAY TRUE);
+
+COPY
+(
+select skip_reason,
+       count(*) as total,
+       sum(size) ::bigint as size
+from 'data/*.parquet'
+where skip_reason != ''
+group by skip_reason
+order by total DESC
+    limit 10
+    ) TO 'stats/skipped_files.json' (ARRAY TRUE);
+
+
+COPY
+(
+select date_trunc('MONTH', uploaded_on) as month,
+           count(distinct project_release)::bigint                 as total_uploads,
+           count(distinct project_name)::bigint                    as project_releases,
+           count(distinct (project_name, project_version))::bigint as total_project_versions,
+           count(*)::bigint                                        as total_files,
+           sum(size)::bigint                                       as total_size,
+           sum(lines)::bigint                                      as total_lines,
+            total_lines / date_diff('hours', min(uploaded_on), max(uploaded_on)) as lines_per_hour,
+            total_size / date_diff('hours', min(uploaded_on), max(uploaded_on)) as bytes_per_hour,
+            total_files / date_diff('hours', min(uploaded_on), max(uploaded_on)) as files_per_hour,
+            total_uploads / date_diff('hours', min(uploaded_on), max(uploaded_on)) as releases_per_hour,
+from 'data/*.parquet'
+where date_part('month', uploaded_on) < date_part('month', current_date())
+group by month
+order by month desc
+    ) TO 'stats/over_time.json' (ARRAY TRUE);
+
+
+COPY
+(
+select date_trunc('MONTH', uploaded_on) as month,
+           count(distinct project_release)::bigint                 as total_uploads,
+           count(distinct project_name)::bigint                    as project_releases,
+           count(distinct (project_name, project_version))::bigint as total_project_versions,
+           count(*)::bigint                                        as files,
+           sum(size)::bigint                                       as size,
+           sum(lines)::bigint                                      as lines,
+from 'data/*.parquet'
+where date_part('month', uploaded_on) < date_part('month', current_date())
+group by month
+order by month
+    ) TO 'stats/over_time.json' (ARRAY TRUE);

stats/general_stats.json

@@ -0,0 +1 @@
+{"total_files":983816449,"unique_files":76513966,"total_size":57132996204260,"total_lines":307727578583}

stats/over_time.json

@@ -0,0 +1,114 @@
+[
+	{"month":"2005-03-01","total_uploads":4,"project_releases":4,"total_project_versions":4,"files":350,"size":3500919,"lines":93451},
+	{"month":"2005-04-01","total_uploads":11,"project_releases":5,"total_project_versions":6,"files":1238,"size":5652555,"lines":158222},
+	{"month":"2005-05-01","total_uploads":7,"project_releases":4,"total_project_versions":5,"files":1663,"size":13180070,"lines":344642},
+	{"month":"2005-06-01","total_uploads":9,"project_releases":4,"total_project_versions":5,"files":205,"size":587933,"lines":15723},
+	{"month":"2006-01-01","total_uploads":113,"project_releases":39,"total_project_versions":59,"files":9633,"size":110521983,"lines":3063385},
+	{"month":"2006-02-01","total_uploads":100,"project_releases":36,"total_project_versions":57,"files":4590,"size":38928049,"lines":800652},
+	{"month":"2006-03-01","total_uploads":113,"project_releases":44,"total_project_versions":66,"files":6366,"size":70288872,"lines":1332691},
+	{"month":"2006-04-01","total_uploads":120,"project_releases":38,"total_project_versions":57,"files":7702,"size":61888670,"lines":1175918},
+	{"month":"2006-05-01","total_uploads":98,"project_releases":38,"total_project_versions":53,"files":7806,"size":64598841,"lines":1412992},
+	{"month":"2006-06-01","total_uploads":176,"project_releases":48,"total_project_versions":73,"files":7239,"size":105488199,"lines":1259086},
+	{"month":"2007-01-01","total_uploads":226,"project_releases":92,"total_project_versions":127,"files":11994,"size":90943905,"lines":1690156},
+	{"month":"2007-02-01","total_uploads":371,"project_releases":130,"total_project_versions":193,"files":24730,"size":135389095,"lines":2604851},
+	{"month":"2007-03-01","total_uploads":463,"project_releases":165,"total_project_versions":239,"files":30536,"size":118252526,"lines":2159283},
+	{"month":"2007-04-01","total_uploads":340,"project_releases":176,"total_project_versions":215,"files":21479,"size":112210673,"lines":2380797},
+	{"month":"2007-05-01","total_uploads":344,"project_releases":129,"total_project_versions":182,"files":26407,"size":142147397,"lines":2387704},
+	{"month":"2007-06-01","total_uploads":329,"project_releases":112,"total_project_versions":168,"files":20652,"size":140897853,"lines":2162067},
+	{"month":"2008-01-01","total_uploads":649,"project_releases":252,"total_project_versions":408,"files":64738,"size":497491246,"lines":6938182},
+	{"month":"2008-02-01","total_uploads":805,"project_releases":297,"total_project_versions":480,"files":55424,"size":505854256,"lines":13593519},
+	{"month":"2008-03-01","total_uploads":827,"project_releases":281,"total_project_versions":484,"files":63844,"size":448980901,"lines":7457976},
+	{"month":"2008-04-01","total_uploads":774,"project_releases":322,"total_project_versions":487,"files":55875,"size":414168842,"lines":7548519},
+	{"month":"2008-05-01","total_uploads":871,"project_releases":314,"total_project_versions":520,"files":60117,"size":511744413,"lines":9799126},
+	{"month":"2008-06-01","total_uploads":789,"project_releases":353,"total_project_versions":500,"files":67535,"size":577370871,"lines":9440531},
+	{"month":"2009-01-01","total_uploads":1194,"project_releases":475,"total_project_versions":864,"files":91311,"size":661471005,"lines":11961197},
+	{"month":"2009-02-01","total_uploads":1143,"project_releases":526,"total_project_versions":807,"files":106622,"size":775243665,"lines":13175312},
+	{"month":"2009-03-01","total_uploads":1351,"project_releases":595,"total_project_versions":988,"files":101659,"size":765708206,"lines":14588614},
+	{"month":"2009-04-01","total_uploads":1142,"project_releases":518,"total_project_versions":873,"files":115121,"size":730403385,"lines":13873609},
+	{"month":"2009-05-01","total_uploads":1281,"project_releases":600,"total_project_versions":967,"files":101966,"size":672791860,"lines":11881388},
+	{"month":"2009-06-01","total_uploads":1321,"project_releases":570,"total_project_versions":973,"files":110403,"size":850588717,"lines":13150039},
+	{"month":"2010-01-01","total_uploads":1483,"project_releases":699,"total_project_versions":1232,"files":125884,"size":1144846176,"lines":14990321},
+	{"month":"2010-02-01","total_uploads":1607,"project_releases":752,"total_project_versions":1281,"files":119021,"size":1182354641,"lines":19764074},
+	{"month":"2010-03-01","total_uploads":1878,"project_releases":852,"total_project_versions":1453,"files":153699,"size":1286982348,"lines":32444134},
+	{"month":"2010-04-01","total_uploads":1771,"project_releases":846,"total_project_versions":1424,"files":168873,"size":1288328343,"lines":22344227},
+	{"month":"2010-05-01","total_uploads":1613,"project_releases":786,"total_project_versions":1339,"files":131175,"size":1287485035,"lines":24631902},
+	{"month":"2010-06-01","total_uploads":1936,"project_releases":826,"total_project_versions":1499,"files":148601,"size":1361944001,"lines":21920662},
+	{"month":"2011-01-01","total_uploads":2118,"project_releases":1015,"total_project_versions":1798,"files":213789,"size":2030355590,"lines":33690337},
+	{"month":"2011-02-01","total_uploads":2333,"project_releases":1057,"total_project_versions":1899,"files":277087,"size":2530760575,"lines":48261000},
+	{"month":"2011-03-01","total_uploads":2333,"project_releases":1082,"total_project_versions":1983,"files":210307,"size":2444950673,"lines":27984536},
+	{"month":"2011-04-01","total_uploads":2340,"project_releases":1185,"total_project_versions":1983,"files":246294,"size":2263084961,"lines":41133611},
+	{"month":"2011-05-01","total_uploads":2556,"project_releases":1212,"total_project_versions":2150,"files":268409,"size":1956385449,"lines":36669124},
+	{"month":"2011-06-01","total_uploads":2617,"project_releases":1186,"total_project_versions":2247,"files":251309,"size":1943980756,"lines":31778205},
+	{"month":"2012-01-01","total_uploads":2732,"project_releases":1349,"total_project_versions":2434,"files":220348,"size":2083202655,"lines":33117399},
+	{"month":"2012-02-01","total_uploads":3268,"project_releases":1440,"total_project_versions":2851,"files":229758,"size":2018196388,"lines":34627254},
+	{"month":"2012-03-01","total_uploads":3779,"project_releases":1689,"total_project_versions":3301,"files":264477,"size":2487350356,"lines":40324310},
+	{"month":"2012-04-01","total_uploads":3521,"project_releases":1650,"total_project_versions":3119,"files":291228,"size":2646190452,"lines":46404847},
+	{"month":"2012-05-01","total_uploads":3578,"project_releases":1664,"total_project_versions":3182,"files":326711,"size":2542462274,"lines":38407594},
+	{"month":"2012-06-01","total_uploads":3506,"project_releases":1658,"total_project_versions":3059,"files":323991,"size":2462543848,"lines":40807834},
+	{"month":"2013-01-01","total_uploads":5249,"project_releases":2323,"total_project_versions":4727,"files":494172,"size":5194803396,"lines":92632646},
+	{"month":"2013-02-01","total_uploads":5212,"project_releases":2344,"total_project_versions":4648,"files":474041,"size":6012436468,"lines":81030332},
+	{"month":"2013-03-01","total_uploads":5982,"project_releases":2596,"total_project_versions":5219,"files":422295,"size":6770915833,"lines":79354997},
+	{"month":"2013-04-01","total_uploads":5397,"project_releases":2487,"total_project_versions":4888,"files":363941,"size":4702703210,"lines":74848745},
+	{"month":"2013-05-01","total_uploads":5797,"project_releases":2584,"total_project_versions":5245,"files":429291,"size":5381222189,"lines":84351704},
+	{"month":"2013-06-01","total_uploads":5095,"project_releases":2358,"total_project_versions":4687,"files":403197,"size":3760319873,"lines":62227126},
+	{"month":"2014-01-01","total_uploads":7787,"project_releases":3132,"total_project_versions":6616,"files":676715,"size":9603151404,"lines":149866740},
+	{"month":"2014-02-01","total_uploads":7034,"project_releases":3228,"total_project_versions":6123,"files":612498,"size":8590266223,"lines":127907180},
+	{"month":"2014-03-01","total_uploads":8340,"project_releases":3380,"total_project_versions":7172,"files":609287,"size":8457692169,"lines":117432007},
+	{"month":"2014-04-01","total_uploads":7976,"project_releases":3402,"total_project_versions":6780,"files":712917,"size":13570376416,"lines":137623793},
+	{"month":"2014-05-01","total_uploads":8709,"project_releases":3399,"total_project_versions":7322,"files":745704,"size":12534246649,"lines":145653008},
+	{"month":"2014-06-01","total_uploads":8911,"project_releases":3473,"total_project_versions":7484,"files":756749,"size":12811014047,"lines":161135685},
+	{"month":"2015-01-01","total_uploads":10524,"project_releases":4049,"total_project_versions":8651,"files":1019778,"size":18105206868,"lines":265555649},
+	{"month":"2015-02-01","total_uploads":11440,"project_releases":4266,"total_project_versions":9260,"files":966734,"size":16814971068,"lines":233997962},
+	{"month":"2015-03-01","total_uploads":13191,"project_releases":4797,"total_project_versions":10496,"files":1177702,"size":22111372044,"lines":273171801},
+	{"month":"2015-04-01","total_uploads":12902,"project_releases":4695,"total_project_versions":10066,"files":973230,"size":17634251586,"lines":242640006},
+	{"month":"2015-05-01","total_uploads":13181,"project_releases":4663,"total_project_versions":9963,"files":979394,"size":23100413882,"lines":252742582},
+	{"month":"2015-06-01","total_uploads":13642,"project_releases":4673,"total_project_versions":10266,"files":995977,"size":31796770500,"lines":344623370},
+	{"month":"2016-01-01","total_uploads":16804,"project_releases":5497,"total_project_versions":12406,"files":1457868,"size":38811316642,"lines":398551391},
+	{"month":"2016-02-01","total_uploads":18384,"project_releases":5912,"total_project_versions":13326,"files":1586955,"size":41108923850,"lines":386766709},
+	{"month":"2016-03-01","total_uploads":19734,"project_releases":6241,"total_project_versions":14682,"files":1822946,"size":36030961317,"lines":450098709},
+	{"month":"2016-04-01","total_uploads":19436,"project_releases":5838,"total_project_versions":13899,"files":1858489,"size":59754105599,"lines":456873137},
+	{"month":"2016-05-01","total_uploads":18349,"project_releases":5848,"total_project_versions":13540,"files":1681295,"size":36254232591,"lines":392161405},
+	{"month":"2016-06-01","total_uploads":18704,"project_releases":5800,"total_project_versions":13512,"files":1820551,"size":36684306807,"lines":381304721},
+	{"month":"2017-01-01","total_uploads":23672,"project_releases":6910,"total_project_versions":16192,"files":2466907,"size":77331406175,"lines":683847787},
+	{"month":"2017-02-01","total_uploads":23677,"project_releases":7028,"total_project_versions":16201,"files":2179973,"size":77867814326,"lines":658289160},
+	{"month":"2017-03-01","total_uploads":26021,"project_releases":7638,"total_project_versions":17880,"files":2490002,"size":89147202195,"lines":774838286},
+	{"month":"2017-04-01","total_uploads":35672,"project_releases":9385,"total_project_versions":27127,"files":3283914,"size":99520366206,"lines":919456884},
+	{"month":"2017-05-01","total_uploads":28321,"project_releases":8914,"total_project_versions":19615,"files":2973793,"size":95334411830,"lines":774003847},
+	{"month":"2017-06-01","total_uploads":28936,"project_releases":8107,"total_project_versions":20365,"files":3109134,"size":108199225039,"lines":858195258},
+	{"month":"2018-01-01","total_uploads":32613,"project_releases":9431,"total_project_versions":22201,"files":3174698,"size":111605675365,"lines":793708417},
+	{"month":"2018-02-01","total_uploads":32475,"project_releases":9513,"total_project_versions":21605,"files":3073154,"size":109346158126,"lines":869140015},
+	{"month":"2018-03-01","total_uploads":38676,"project_releases":10599,"total_project_versions":25632,"files":3545966,"size":147984155626,"lines":1119136880},
+	{"month":"2018-04-01","total_uploads":41192,"project_releases":10868,"total_project_versions":27811,"files":3655738,"size":183283280647,"lines":1061704510},
+	{"month":"2018-05-01","total_uploads":38060,"project_releases":10270,"total_project_versions":24701,"files":3999846,"size":189869806693,"lines":1191359848},
+	{"month":"2018-06-01","total_uploads":41530,"project_releases":12670,"total_project_versions":27164,"files":3910673,"size":162540651242,"lines":1151734386},
+	{"month":"2019-01-01","total_uploads":52038,"project_releases":11825,"total_project_versions":30187,"files":4920995,"size":309275527088,"lines":2026504096},
+	{"month":"2019-02-01","total_uploads":51305,"project_releases":12189,"total_project_versions":29895,"files":4900008,"size":298373104048,"lines":1899050690},
+	{"month":"2019-03-01","total_uploads":56259,"project_releases":13381,"total_project_versions":33409,"files":5361391,"size":327286308147,"lines":3010070614},
+	{"month":"2019-04-01","total_uploads":59822,"project_releases":14961,"total_project_versions":35209,"files":5739563,"size":427699036960,"lines":2075094954},
+	{"month":"2019-05-01","total_uploads":58913,"project_releases":13604,"total_project_versions":34587,"files":5622633,"size":374746803287,"lines":1973347337},
+	{"month":"2019-06-01","total_uploads":53873,"project_releases":12911,"total_project_versions":31493,"files":5905838,"size":428987119398,"lines":2030456144},
+	{"month":"2020-01-01","total_uploads":77420,"project_releases":15564,"total_project_versions":45084,"files":7081415,"size":488070883330,"lines":2287390298},
+	{"month":"2020-02-01","total_uploads":78835,"project_releases":16068,"total_project_versions":45401,"files":8525553,"size":398270250755,"lines":2475990886},
+	{"month":"2020-03-01","total_uploads":91678,"project_releases":17502,"total_project_versions":54530,"files":8119484,"size":374431078920,"lines":2695840238},
+	{"month":"2020-04-01","total_uploads":100277,"project_releases":18863,"total_project_versions":56761,"files":9886158,"size":458217908889,"lines":2833095272},
+	{"month":"2020-05-01","total_uploads":104739,"project_releases":19646,"total_project_versions":58347,"files":10713111,"size":673221916531,"lines":2995732406},
+	{"month":"2020-06-01","total_uploads":112078,"project_releases":19550,"total_project_versions":60699,"files":11328177,"size":519618677325,"lines":2918256286},
+	{"month":"2021-01-01","total_uploads":128025,"project_releases":20872,"total_project_versions":64623,"files":12553637,"size":712267346049,"lines":3993116531},
+	{"month":"2021-02-01","total_uploads":133506,"project_releases":23133,"total_project_versions":68128,"files":13061555,"size":603417611252,"lines":4156288238},
+	{"month":"2021-03-01","total_uploads":157409,"project_releases":24286,"total_project_versions":80532,"files":16826646,"size":851922177092,"lines":5623018580},
+	{"month":"2021-04-01","total_uploads":144194,"project_releases":22450,"total_project_versions":73061,"files":13647561,"size":795759022187,"lines":5793234173},
+	{"month":"2021-05-01","total_uploads":147313,"project_releases":22327,"total_project_versions":73219,"files":13480241,"size":800724130090,"lines":5135907063},
+	{"month":"2021-06-01","total_uploads":140241,"project_releases":21657,"total_project_versions":70760,"files":13892225,"size":767921726945,"lines":4660408596},
+	{"month":"2022-01-01","total_uploads":154642,"project_releases":23928,"total_project_versions":75453,"files":14962968,"size":905644701839,"lines":4890799032},
+	{"month":"2022-02-01","total_uploads":145124,"project_releases":24023,"total_project_versions":71323,"files":21642235,"size":1442991975859,"lines":6953243787},
+	{"month":"2022-03-01","total_uploads":172259,"project_releases":26113,"total_project_versions":84273,"files":24674207,"size":1766174594234,"lines":8697920926},
+	{"month":"2022-04-01","total_uploads":160321,"project_releases":24337,"total_project_versions":78255,"files":24047745,"size":1759102661711,"lines":8123411762},
+	{"month":"2022-05-01","total_uploads":171802,"project_releases":25118,"total_project_versions":84020,"files":26055412,"size":1923448893739,"lines":8772530612},
+	{"month":"2022-06-01","total_uploads":165003,"project_releases":24663,"total_project_versions":80265,"files":26862013,"size":1924843587584,"lines":9102353251},
+	{"month":"2023-01-01","total_uploads":213193,"project_releases":28722,"total_project_versions":95416,"files":31850863,"size":2286085347832,"lines":9696136189},
+	{"month":"2023-02-01","total_uploads":210528,"project_releases":30281,"total_project_versions":97577,"files":27434766,"size":1920162245428,"lines":8318250783},
+	{"month":"2023-03-01","total_uploads":236190,"project_releases":31442,"total_project_versions":106934,"files":27608428,"size":1916834729049,"lines":8490952683},
+	{"month":"2023-04-01","total_uploads":214162,"project_releases":29994,"total_project_versions":95548,"files":27265712,"size":1611737815616,"lines":7477192944},
+	{"month":"2023-05-01","total_uploads":222293,"project_releases":30891,"total_project_versions":100494,"files":27030193,"size":1772029420428,"lines":8090404983},
+	{"month":"2023-06-01","total_uploads":217795,"project_releases":30253,"total_project_versions":99280,"files":24912299,"size":1708307468520,"lines":7130514565}
+]