diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 47eb225..3f2869d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: uses: erlef/setup-beam@v1 with: otp-version: '26.0.2' - gleam-version: '1.2.1' + gleam-version: '1.4.1' rebar3-version: '3' # elixir-version: "1.15.4" - name: Download gleam dependencies @@ -51,7 +51,7 @@ jobs: uses: erlef/setup-beam@v1 with: otp-version: '26.0.2' - gleam-version: '1.2.1' + gleam-version: '1.4.1' rebar3-version: '3' - name: Download gleam dependencies run: gleam deps download diff --git a/.mise.toml b/.mise.toml index b7e6878..a9ab64a 100644 --- a/.mise.toml +++ b/.mise.toml @@ -1,6 +1,6 @@ # Runtimes used by the stack. [tools] -gleam = "1.4.0" +gleam = "1.4.1" erlang = "27" node = "22" deno = "latest" diff --git a/Dockerfile b/Dockerfile index 1e4a346..cd9ff13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=x86_64 ghcr.io/gleam-lang/gleam:v1.4.0-erlang-alpine AS builder +FROM --platform=x86_64 ghcr.io/gleam-lang/gleam:v1.4.1-erlang-alpine AS builder RUN apk add build-base ca-certificates RUN mkdir -p /build/backend/src diff --git a/apps/backend/db/migrations/20240902224247_create_analytics_timeseries.sql b/apps/backend/db/migrations/20240902224247_create_analytics_timeseries.sql new file mode 100644 index 0000000..97de598 --- /dev/null +++ b/apps/backend/db/migrations/20240902224247_create_analytics_timeseries.sql @@ -0,0 +1,10 @@ +-- migrate:up +create table analytics_timeseries ( + query text not null, + occurences int not null, + date timestamptz not null, + primary key (query, date) +); + +-- migrate:down +drop table analytics_timeseries; diff --git a/apps/backend/db/migrations/20240902225236_alter_default_value_in_search_analytics.sql b/apps/backend/db/migrations/20240902225236_alter_default_value_in_search_analytics.sql new file mode 100644 index 0000000..49216fc --- /dev/null +++ b/apps/backend/db/migrations/20240902225236_alter_default_value_in_search_analytics.sql @@ -0,0 +1,7 @@ +-- migrate:up +alter table only search_analytics alter column occurences set default 1; +update search_analytics set occurences = occurences + 1; + +-- migrate:down +alter table only search_analytics alter column occurences set default 0; +update search_analytics set occurences = occurences - 1; diff --git a/apps/backend/db/schema.sql b/apps/backend/db/schema.sql index 73ef95e..f767afe 100644 --- a/apps/backend/db/schema.sql +++ b/apps/backend/db/schema.sql @@ -96,6 +96,17 @@ ALTER TABLE public.analytics ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY ( ); +-- +-- Name: analytics_timeseries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.analytics_timeseries ( + query text NOT NULL, + occurences integer NOT NULL, + date timestamp with time zone NOT NULL +); + + -- -- Name: hex_read; Type: TABLE; Schema: public; Owner: - -- @@ -304,7 +315,7 @@ CREATE TABLE public.schema_migrations ( CREATE TABLE public.search_analytics ( query text NOT NULL, - occurences integer DEFAULT 0 NOT NULL, + occurences integer DEFAULT 1 NOT NULL, created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL ); @@ -326,6 +337,14 @@ ALTER TABLE ONLY public.analytics ADD CONSTRAINT analytics_pkey PRIMARY KEY (id); +-- +-- Name: analytics_timeseries analytics_timeseries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.analytics_timeseries + ADD CONSTRAINT analytics_timeseries_pkey PRIMARY KEY (query, date); + + -- -- Name: hex_read hex_read_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -590,4 +609,6 @@ INSERT INTO public.schema_migrations (version) VALUES ('20240521204341'), ('20240801164720'), ('20240801211520'), - ('20240801220817'); + ('20240801220817'), + ('20240902224247'), + ('20240902225236'); diff --git a/apps/backend/gleam.toml b/apps/backend/gleam.toml index f6e9f85..3a3c8df 100644 --- a/apps/backend/gleam.toml +++ b/apps/backend/gleam.toml @@ -4,7 +4,9 @@ version = "1.0.0" [dependencies] aws4_request = ">= 0.1.1 and < 1.0.0" birl = "~> 1.6" -dot_env = "~> 0.5" +chomp = ">= 0.1.0 and < 1.0.0" +cors_builder = ">= 2.0.0 and < 3.0.0" +decipher = ">= 1.2.0 and < 2.0.0" gleam_erlang = "~> 0.25" gleam_hexpm = "~> 1.0" gleam_http = "~> 3.6" @@ -14,19 +16,16 @@ gleam_otp = "~> 0.10" gleam_package_interface = ">= 1.0.0 and < 2.0.0" gleam_pgo = {path = "../../packages/pgo"} gleam_stdlib = "~> 0.34 or ~> 1.0" -mist = ">= 1.0.0 and < 2.0.0" +glexer = ">= 1.0.1 and < 2.0.0" +mist = ">= 1.2.0 and < 2.0.0" pgo = "~> 0.14" prng = ">= 3.0.3 and < 4.0.0" radiate = ">= 0.4.0 and < 1.0.0" ranger = ">= 1.2.0 and < 2.0.0" -simplifile = ">= 1.7.0 and < 2.0.0" +simplifile = ">= 2.1.0 and < 3.0.0" tom = { path ="../../packages/tom" } verl = ">= 1.1.1 and < 2.0.0" -wisp = "~> 0.14" -cors_builder = ">= 1.0.0 and < 2.0.0" -decipher = ">= 1.2.0 and < 2.0.0" -chomp = ">= 0.1.0 and < 1.0.0" -glexer = ">= 1.0.1 and < 2.0.0" +wisp = ">= 1.1.0 and < 2.0.0" [dev-dependencies] gleeunit = "~> 1.0" diff --git a/apps/backend/manifest.toml b/apps/backend/manifest.toml index df2073a..fe50b78 100644 --- a/apps/backend/manifest.toml +++ b/apps/backend/manifest.toml @@ -6,9 +6,10 @@ packages = [ { name = "backoff", version = "1.1.6", build_tools = ["rebar3"], requirements = [], otp_app = "backoff", source = "hex", outer_checksum = "CF0CFFF8995FB20562F822E5CC47D8CCF664C5ECDC26A684CBE85C225F9D7C39" }, { name = "birl", version = "1.7.1", build_tools = ["gleam"], requirements = ["gleam_stdlib", "ranger"], otp_app = "birl", source = "hex", outer_checksum = "5C66647D62BCB11FE327E7A6024907C4A17954EF22865FE0940B54A852446D01" }, { name = "chomp", version = "0.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "chomp", source = "hex", outer_checksum = "C87304897B4D4DEA69420DB2FF88B087673AAE9EC09CA8A0FBF4675F605767C2" }, - { name = "cors_builder", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_stdlib", "mist", "wisp"], otp_app = "cors_builder", source = "hex", outer_checksum = "951B5B648E958BD6181A6EED98BCA4EEB302B83DC7DCE2954B3462114209EC43" }, + { name = "cors_builder", version = "2.0.0", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_stdlib", "mist", "wisp"], otp_app = "cors_builder", source = "hex", outer_checksum = "E4634D1CAD89BC26B4FE986B45D55268BA5369E9224C2FD7872C0DB5AC028E47" }, { name = "decipher", version = "1.2.0", build_tools = ["gleam"], requirements = ["birl", "gleam_json", "gleam_stdlib", "stoiridh_version"], otp_app = "decipher", source = "hex", outer_checksum = "9F1B5C6FF0D798046E4E0EF87D09DD729324CB72BD7F0D4152B797324D51223E" }, - { name = "dot_env", version = "0.5.1", build_tools = ["gleam"], requirements = ["gleam_stdlib", "simplifile"], otp_app = "dot_env", source = "hex", outer_checksum = "AF5C972D6129F67AF3BB00134AB2808D37111A8D61686CFA86F3ADF652548982" }, + { name = "directories", version = "1.1.0", build_tools = ["gleam"], requirements = ["envoy", "gleam_stdlib", "platform", "simplifile"], otp_app = "directories", source = "hex", outer_checksum = "BDA521A4EB9EE3A7894F0DC863797878E91FF5C7826F7084B2E731E208BDB076" }, + { name = "envoy", version = "1.0.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "envoy", source = "hex", outer_checksum = "CFAACCCFC47654F7E8B75E614746ED924C65BD08B1DE21101548AC314A8B6A41" }, { name = "exception", version = "2.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "exception", source = "hex", outer_checksum = "F5580D584F16A20B7FCDCABF9E9BE9A2C1F6AC4F9176FA6DD0B63E3B20D450AA" }, { name = "filepath", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "EFB6FF65C98B2A16378ABC3EE2B14124168C0CE5201553DE652E2644DCFDB594" }, { name = "filespy", version = "0.5.0", build_tools = ["gleam"], requirements = ["fs", "gleam_erlang", "gleam_otp", "gleam_stdlib"], otp_app = "filespy", source = "hex", outer_checksum = "F8E7A9C9CA86D68CCC25491125BFF36BEF7483892D7BEC24AA30D6B540504F06" }, @@ -30,33 +31,33 @@ packages = [ { name = "glisten", version = "2.0.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_otp", "gleam_stdlib"], otp_app = "glisten", source = "hex", outer_checksum = "CF3A9383E9BA4A8CBAF2F7B799716290D02F2AC34E7A77556B49376B662B9314" }, { name = "gramps", version = "2.0.3", build_tools = ["gleam"], requirements = ["gleam_crypto", "gleam_erlang", "gleam_http", "gleam_stdlib"], otp_app = "gramps", source = "hex", outer_checksum = "3CCAA6E081225180D95C79679D383BBF51C8D1FDC1B84DA1DA444F628C373793" }, { name = "hpack_erl", version = "0.3.0", build_tools = ["rebar3"], requirements = [], otp_app = "hpack", source = "hex", outer_checksum = "D6137D7079169D8C485C6962DFE261AF5B9EF60FBC557344511C1E65E3D95FB0" }, - { name = "logging", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "logging", source = "hex", outer_checksum = "FCB111401BDB4703A440A94FF8CC7DA521112269C065F219C2766998333E7738" }, + { name = "logging", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "logging", source = "hex", outer_checksum = "1098FBF10B54B44C2C7FDF0B01C1253CAFACDACABEFB4B0D027803246753E06D" }, { name = "marceau", version = "1.2.0", build_tools = ["gleam"], requirements = [], otp_app = "marceau", source = "hex", outer_checksum = "5188D643C181EE350D8A20A3BDBD63AF7B6C505DE333CFBE05EF642ADD88A59B" }, { name = "mist", version = "1.2.0", build_tools = ["gleam"], requirements = ["birl", "gleam_erlang", "gleam_http", "gleam_otp", "gleam_stdlib", "glisten", "gramps", "hpack_erl", "logging"], otp_app = "mist", source = "hex", outer_checksum = "109B4D64E68C104CC23BB3CC5441ECD479DD7444889DA01113B75C6AF0F0E17B" }, { name = "opentelemetry_api", version = "1.3.0", build_tools = ["rebar3", "mix"], requirements = ["opentelemetry_semantic_conventions"], otp_app = "opentelemetry_api", source = "hex", outer_checksum = "B9E5FF775FD064FA098DBA3C398490B77649A352B40B0B730A6B7DC0BDD68858" }, { name = "opentelemetry_semantic_conventions", version = "0.2.0", build_tools = ["rebar3", "mix"], requirements = [], otp_app = "opentelemetry_semantic_conventions", source = "hex", outer_checksum = "D61FA1F5639EE8668D74B527E6806E0503EFC55A42DB7B5F39939D84C07D6895" }, { name = "pg_types", version = "0.4.0", build_tools = ["rebar3"], requirements = [], otp_app = "pg_types", source = "hex", outer_checksum = "B02EFA785CAECECF9702C681C80A9CA12A39F9161A846CE17B01FB20AEEED7EB" }, { name = "pgo", version = "0.14.0", build_tools = ["rebar3"], requirements = ["backoff", "opentelemetry_api", "pg_types"], otp_app = "pgo", source = "hex", outer_checksum = "71016C22599936E042DC0012EE4589D24C71427D266292F775EBF201D97DF9C9" }, + { name = "platform", version = "1.0.0", build_tools = ["gleam"], requirements = [], otp_app = "platform", source = "hex", outer_checksum = "8339420A95AD89AAC0F82F4C3DB8DD401041742D6C3F46132A8739F6AEB75391" }, { name = "pprint", version = "1.0.3", build_tools = ["gleam"], requirements = ["glam", "gleam_stdlib"], otp_app = "pprint", source = "hex", outer_checksum = "76BBB92E23D12D954BD452686543F29EDE8EBEBB7FC0ACCBCA66EEF276EC3A06" }, { name = "prng", version = "3.0.3", build_tools = ["gleam"], requirements = ["gleam_bitwise", "gleam_stdlib"], otp_app = "prng", source = "hex", outer_checksum = "53006736FE23A0F61828C95B505193E10905D8DB76E128F1642D3E571E08F589" }, { name = "radiate", version = "0.4.0", build_tools = ["gleam"], requirements = ["filespy", "gleam_otp", "gleam_stdlib", "shellout"], otp_app = "radiate", source = "hex", outer_checksum = "93A76A66EE4741DBFD3E79E27CBD11FE58EC3CB1C58F017FC165944E339D6293" }, { name = "ranger", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "ranger", source = "hex", outer_checksum = "1566C272B1D141B3BBA38B25CB761EF56E312E79EC0E2DFD4D3C19FB0CC1F98C" }, { name = "shellout", version = "1.6.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "shellout", source = "hex", outer_checksum = "E2FCD18957F0E9F67E1F497FC9FF57393392F8A9BAEAEA4779541DE7A68DD7E0" }, - { name = "simplifile", version = "1.7.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "1D5DFA3A2F9319EC85825F6ED88B8E449F381B0D55A62F5E61424E748E7DDEB0" }, + { name = "simplifile", version = "2.1.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "BDD04F5D31D6D34E2EDFAEF0B68A6297AEC939888C3BFCE61133DE13857F6DA2" }, { name = "stoiridh_version", version = "0.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "stoiridh_version", source = "hex", outer_checksum = "EEF8ADAB9755BD33EB202F169376F1A7797AEF90823FDCA671D8590D04FBF56B" }, { name = "thoas", version = "1.2.1", build_tools = ["rebar3"], requirements = [], otp_app = "thoas", source = "hex", outer_checksum = "E38697EDFFD6E91BD12CEA41B155115282630075C2A727E7A6B2947F5408B86A" }, { name = "tom", version = "0.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], source = "local", path = "../../packages/tom" }, { name = "verl", version = "1.1.1", build_tools = ["rebar3"], requirements = [], otp_app = "verl", source = "hex", outer_checksum = "0925E51CD92A0A8BE271765B02430B2E2CFF8AC30EF24D123BD0D58511E8FB18" }, - { name = "wisp", version = "0.14.0", build_tools = ["gleam"], requirements = ["exception", "gleam_crypto", "gleam_erlang", "gleam_http", "gleam_json", "gleam_stdlib", "logging", "marceau", "mist", "simplifile"], otp_app = "wisp", source = "hex", outer_checksum = "9F5453AF1F9275E6F8707BC815D6A6A9DF41551921B16FBDBA52883773BAE684" }, + { name = "wisp", version = "1.1.0", build_tools = ["gleam"], requirements = ["directories", "exception", "gleam_crypto", "gleam_erlang", "gleam_http", "gleam_json", "gleam_stdlib", "logging", "marceau", "mist", "simplifile"], otp_app = "wisp", source = "hex", outer_checksum = "5B25E37E08A94039668D2ACF7B2D2A665BEAF4B48EFA8613010B4E3164137664" }, ] [requirements] aws4_request = { version = ">= 0.1.1 and < 1.0.0" } birl = { version = "~> 1.6" } chomp = { version = ">= 0.1.0 and < 1.0.0" } -cors_builder = { version = ">= 1.0.0 and < 2.0.0" } +cors_builder = { version = ">= 2.0.0 and < 3.0.0" } decipher = { version = ">= 1.2.0 and < 2.0.0" } -dot_env = { version = "~> 0.5" } gleam_erlang = { version = "~> 0.25" } gleam_hexpm = { version = "~> 1.0" } gleam_http = { version = "~> 3.6" } @@ -68,13 +69,13 @@ gleam_pgo = { path = "../../packages/pgo" } gleam_stdlib = { version = "~> 0.34 or ~> 1.0" } gleeunit = { version = "~> 1.0" } glexer = { version = ">= 1.0.1 and < 2.0.0" } -mist = { version = ">= 1.0.0 and < 2.0.0" } +mist = { version = ">= 1.2.0 and < 2.0.0" } pgo = { version = "~> 0.14" } pprint = { version = ">= 1.0.3 and < 2.0.0" } prng = { version = ">= 3.0.3 and < 4.0.0" } radiate = { version = ">= 0.4.0 and < 1.0.0" } ranger = { version = ">= 1.2.0 and < 2.0.0" } -simplifile = { version = ">= 1.7.0 and < 2.0.0" } +simplifile = { version = ">= 2.1.0 and < 3.0.0" } tom = { path = "../../packages/tom" } verl = { version = ">= 1.1.1 and < 2.0.0" } -wisp = { version = "~> 0.14" } +wisp = { version = ">= 1.1.0 and < 2.0.0" } diff --git a/apps/backend/src/api/github.gleam b/apps/backend/src/api/github.gleam index becbc3c..a00f0d8 100644 --- a/apps/backend/src/api/github.gleam +++ b/apps/backend/src/api/github.gleam @@ -25,7 +25,7 @@ fn query( use response <- result.try( request.new() |> request.set_header("authorization", "Bearer " <> token) - |> request.set_header("user-agent", "gloogle / 0.0.0") + |> request.set_header("user-agent", "gloogle / 1.0.0") |> request.set_method(http.Post) |> request.set_scheme(http.Https) |> request.set_host("api.github.com") diff --git a/apps/backend/src/api/github/stargazer_count.gleam b/apps/backend/src/api/github/stargazer_count.gleam index 9cfc58d..dd2abb0 100644 --- a/apps/backend/src/api/github/stargazer_count.gleam +++ b/apps/backend/src/api/github/stargazer_count.gleam @@ -15,7 +15,7 @@ pub fn decoder(dyn) { } pub fn variables(name: String, owner: String) { - Some( - json.object([#("name", json.string(name)), #("owner", json.string(owner))]), - ) + let name = json.string(name) + let owner = json.string(owner) + Some(json.object([#("name", name), #("owner", owner)])) } diff --git a/apps/backend/src/api/hex.gleam b/apps/backend/src/api/hex.gleam index 75766ce..937c84e 100644 --- a/apps/backend/src/api/hex.gleam +++ b/apps/backend/src/api/hex.gleam @@ -9,15 +9,15 @@ import gleam/result import gleam/uri pub fn get_package_owners(package_name: String, secret hex_api_key: String) { - use response <- result.try( + use response <- result.try({ request.new() |> request.set_host("hex.pm") |> request.set_path("/api/packages/" <> package_name <> "/owners") |> request.prepend_header("authorization", hex_api_key) - |> request.prepend_header("user-agent", "gloogle / 0.0.0") + |> request.prepend_header("user-agent", "gloogle / 1.0.0") |> httpc.send() - |> result.map_error(error.FetchError), - ) + |> result.map_error(error.FetchError) + }) response.body |> json.decode(using: dynamic.list(decode_hex_owner)) @@ -25,15 +25,15 @@ pub fn get_package_owners(package_name: String, secret hex_api_key: String) { } pub fn get_package(package_name: String, secret hex_api_key: String) { - use response <- result.try( + use response <- result.try({ request.new() |> request.set_host("hex.pm") |> request.set_path("/api/packages/" <> package_name) |> request.prepend_header("authorization", hex_api_key) - |> request.prepend_header("user-agent", "gloogle / 0.0.0") + |> request.prepend_header("user-agent", "gloogle / 1.0.0") |> httpc.send() - |> result.map_error(error.FetchError), - ) + |> result.map_error(error.FetchError) + }) response.body |> json.decode(using: hexpm.decode_package) @@ -52,15 +52,15 @@ fn decode_hex_owner(data) { pub fn lookup_release(release: hexpm.PackageRelease, secret hex_api_key: String) { let assert Ok(url) = uri.parse(release.url) - use response <- result.try( + use response <- result.try({ request.new() |> request.set_host("hex.pm") |> request.set_path(url.path) |> request.prepend_header("authorization", hex_api_key) - |> request.prepend_header("user-agent", "gloogle / 0.0.0") + |> request.prepend_header("user-agent", "gloogle / 1.0.0") |> httpc.send() - |> result.map_error(error.FetchError), - ) + |> result.map_error(error.FetchError) + }) response.body |> json.decode(using: hexpm.decode_release) @@ -68,19 +68,17 @@ pub fn lookup_release(release: hexpm.PackageRelease, secret hex_api_key: String) } pub fn get_api_packages_page(page: Int, hex_api_key: String) { - use response <- result.try( + let page = int.to_string(page) + use response <- result.try({ request.new() |> request.set_host("hex.pm") |> request.set_path("/api/packages") - |> request.set_query([ - #("sort", "updated_at"), - #("page", int.to_string(page)), - ]) + |> request.set_query([#("sort", "updated_at"), #("page", page)]) |> request.prepend_header("authorization", hex_api_key) - |> request.prepend_header("user-agent", "gloogle / 0.0.0") + |> request.prepend_header("user-agent", "gloogle / 1.0.0") |> httpc.send() - |> result.map_error(error.FetchError), - ) + |> result.map_error(error.FetchError) + }) response.body |> json.decode(using: dynamic.list(of: hexpm.decode_package)) diff --git a/apps/backend/src/backend.gleam b/apps/backend/src/backend.gleam index 288fdf0..a9f76dc 100644 --- a/apps/backend/src/backend.gleam +++ b/apps/backend/src/backend.gleam @@ -2,7 +2,6 @@ import backend/config import backend/gleam/type_search/state as type_search import backend/postgres/postgres import backend/router -import dot_env import gleam/erlang/process import gleam/function import gleam/otp/supervisor @@ -12,12 +11,13 @@ import setup import tasks/hex import tasks/popularity import tasks/ranking +import tasks/timeseries import wisp import wisp/logger +import wisp/wisp_mist pub fn main() { wisp.configure_logger() - dot_env.load() let secret_key_base = config.get_secret_key_base() let cnf = config.read_config() @@ -27,37 +27,39 @@ pub fn main() { setup.radiate() let assert Ok(subject) = type_search.init(ctx.db) - // let assert Ok(_) = - // supervisor.start(fn(children) { - // use _ <- function.tap(children) - // supervisor.add(children, { supervisor.worker(fn(_) { Ok(subject) }) }) - // }) let ctx = ctx |> config.add_type_search_subject(subject) let assert Ok(_) = router.handle_request(_, ctx) - |> wisp.mist_handler(secret_key_base) + |> wisp_mist.handler(secret_key_base) |> mist.new() |> mist.port(cnf.port) |> mist.start_http() - let assert Ok(_) = - supervisor.start(fn(periodic_children) { - use _ <- function.tap(periodic_children) - let assert Ok(_) = - supervisor.start(fn(children) { - add_periodic_worker(periodic_children, waiting: 6 * 1000, do: fn() { - hex.sync_new_gleam_releases(ctx, children) - }) - add_periodic_worker(periodic_children, waiting: 86_400_000, do: fn() { - ranking.compute_ranking(ctx) - }) - add_periodic_worker(periodic_children, waiting: 86_400_000, do: fn() { - popularity.compute_popularity(ctx) - }) - }) - }) + let assert Ok(_) = { + use periodic_children <- supervisor.start() + use periodic_children <- function.tap(periodic_children) + let assert Ok(_) = { + use children <- supervisor.start() + // Every 10 seconds + add_periodic_worker(periodic_children, waiting: 10 * 1000, do: fn() { + hex.sync_new_gleam_releases(ctx, children) + }) + // Every day + add_periodic_worker(periodic_children, waiting: 86_400_000, do: fn() { + ranking.compute_ranking(ctx) + }) + // Every day + add_periodic_worker(periodic_children, waiting: 86_400_000, do: fn() { + popularity.compute_popularity(ctx) + }) + // Every hour + add_periodic_worker(periodic_children, waiting: 3600 * 1000, do: fn() { + timeseries.store_timeseries(ctx) + }) + } + } process.sleep_forever() } diff --git a/apps/backend/src/backend/error.gleam b/apps/backend/src/backend/error.gleam index c5677e7..2f436af 100644 --- a/apps/backend/src/backend/error.gleam +++ b/apps/backend/src/backend/error.gleam @@ -164,7 +164,7 @@ pub fn log_simplifile(error: simplifile.FileError) { simplifile.Etxtbsy -> wisp.log_warning("Etxtbsy") simplifile.Exdev -> wisp.log_warning("Exdev") simplifile.NotUtf8 -> wisp.log_warning("NotUtf8") - simplifile.Unknown -> wisp.log_warning("Unknown") + simplifile.Unknown(_) -> wisp.log_warning("Unknown") } } diff --git a/apps/backend/src/backend/postgres/queries.gleam b/apps/backend/src/backend/postgres/queries.gleam index 6b0257e..b2dc20f 100644 --- a/apps/backend/src/backend/postgres/queries.gleam +++ b/apps/backend/src/backend/postgres/queries.gleam @@ -8,6 +8,7 @@ import gleam/dict.{type Dict} import gleam/dynamic import gleam/hexpm import gleam/int +import gleam/io import gleam/json import gleam/list import gleam/option.{type Option, None, Some} @@ -71,6 +72,60 @@ pub fn upsert_search_analytics(db: pgo.Connection, query: String) { }) } +pub fn select_last_day_search_analytics(db: pgo.Connection) { + let #(date, _) = birl.to_erlang_universal_datetime(birl.now()) + let now = birl.from_erlang_universal_datetime(#(date, #(0, 0, 0))) + "SELECT query, occurences + FROM search_analytics + WHERE updated_at >= $1" + |> pgo.execute( + db, + [helpers.convert_time(now)], + dynamic.tuple2(dynamic.string, dynamic.int), + ) + |> result.map(fn(r) { r.rows }) + |> result.map_error(error.DatabaseError) +} + +pub fn upsert_search_analytics_timeseries( + db: pgo.Connection, + analytic: #(String, Int), +) { + let #(date, _) = birl.to_erlang_universal_datetime(birl.now()) + let now = birl.from_erlang_universal_datetime(#(date, #(0, 0, 0))) + let timestamp = helpers.convert_time(now) + let #(query, occurences) = analytic + "INSERT INTO analytics_timeseries (query, occurences, date) + VALUES ($1, $2, $3) + ON CONFLICT (query, date) DO UPDATE + SET occurences = $2" + |> pgo.execute( + db, + [pgo.text(query), pgo.int(occurences), timestamp], + dynamic.dynamic, + ) + |> result.map_error(error.DatabaseError) +} + +pub fn get_timeseries_count(db: pgo.Connection) { + "SELECT + SUM(at.occurences - COALESCE( + (SELECT att.occurences + FROM analytics_timeseries att + WHERE att.date < at.date + AND att.query = at.query), + 0) + ) searches, + at.date date + FROM analytics_timeseries at + WHERE at.date >= now() - INTERVAL '30 day' + GROUP BY at.date + ORDER BY date DESC" + |> pgo.execute(db, [], dynamic.tuple2(dynamic.int, helpers.decode_time)) + |> result.map_error(error.DatabaseError) + |> result.map(fn(r) { r.rows }) +} + pub fn upsert_hex_user(db: pgo.Connection, owner: hexpm.PackageOwner) { let username = pgo.text(owner.username) let email = pgo.nullable(pgo.text, owner.email) @@ -102,6 +157,27 @@ fn get_current_package_owners(db: pgo.Connection, package_id: Int) { |> result.map_error(error.DatabaseError) } +pub fn get_total_searches(db: pgo.Connection) { + "SELECT SUM(occurences) FROM search_analytics" + |> pgo.execute(db, [], dynamic.element(0, dynamic.int)) + |> result.map(fn(r) { r.rows }) + |> result.map_error(error.DatabaseError) +} + +pub fn get_total_signatures(db: pgo.Connection) { + "SELECT COUNT(*) FROM package_type_fun_signature" + |> pgo.execute(db, [], dynamic.element(0, dynamic.int)) + |> result.map(fn(r) { r.rows }) + |> result.map_error(error.DatabaseError) +} + +pub fn get_total_packages(db: pgo.Connection) { + "SELECT COUNT(*) FROM package" + |> pgo.execute(db, [], dynamic.element(0, dynamic.int)) + |> result.map(fn(r) { r.rows }) + |> result.map_error(error.DatabaseError) +} + fn add_new_package_owners( db: pgo.Connection, owners: List(HexUser), diff --git a/apps/backend/src/backend/router.gleam b/apps/backend/src/backend/router.gleam index e765b8c..652054c 100644 --- a/apps/backend/src/backend/router.gleam +++ b/apps/backend/src/backend/router.gleam @@ -4,6 +4,7 @@ import backend/error import backend/gleam/type_search/msg as type_search import backend/postgres/queries import backend/web +import birl import cors_builder as cors import gleam/erlang/process import gleam/http @@ -11,6 +12,7 @@ import gleam/int import gleam/json import gleam/list import gleam/option +import gleam/pair import gleam/result import gleam/string_builder import tasks/hex as syncing @@ -141,6 +143,38 @@ pub fn handle_get(req: Request, ctx: Context) { |> wisp.json_response(200) }) |> result.unwrap(wisp.internal_server_error()) + ["analytics"] -> { + { + use timeseries <- result.try(queries.get_timeseries_count(ctx.db)) + use total <- result.try(queries.get_total_searches(ctx.db)) + use signatures <- result.try(queries.get_total_signatures(ctx.db)) + use packages <- result.try(queries.get_total_packages(ctx.db)) + let total = list.first(total) |> result.unwrap(0) + let signatures = list.first(signatures) |> result.unwrap(0) + let packages = list.first(packages) |> result.unwrap(0) + Ok(#(timeseries, total, signatures, packages)) + } + |> result.map(fn(content) { + let #(timeseries, total, signatures, packages) = content + json.object([ + #("total", json.int(total)), + #("signatures", json.int(signatures)), + #("packages", json.int(packages)), + #("timeseries", { + json.array(timeseries, fn(row) { + json.object([ + #("count", json.int(row.0)), + #("date", json.string(birl.to_iso8601(row.1))), + ]) + }) + }), + ]) + |> json.to_string_builder + |> wisp.json_response(200) + }) + |> result.map_error(error.debug_log) + |> result.unwrap(wisp.internal_server_error()) + } ["search"] -> { wisp.get_query(req) |> list.find(fn(item) { item.0 == "q" }) diff --git a/apps/backend/src/backend/web.gleam b/apps/backend/src/backend/web.gleam index 6eb0820..e7b117e 100644 --- a/apps/backend/src/backend/web.gleam +++ b/apps/backend/src/backend/web.gleam @@ -1,5 +1,5 @@ import backend/config -import cors_builder as cors +import cors_builder as cors_ import gleam/http import wisp.{type Request, type Response} @@ -15,20 +15,20 @@ pub fn foundations(req: Request, handler: Handler) -> Response { pub fn cors() { let origin = case config.is_dev() { - True -> cors.allow_origin(_, "http://localhost:5173") + True -> cors_.allow_origin(_, "http://localhost:5173") False -> fn(cors) { cors - |> cors.allow_origin("https://gloogle.run") - |> cors.allow_origin("https://www.gloogle.run") + |> cors_.allow_origin("https://gloogle.run") + |> cors_.allow_origin("https://www.gloogle.run") } } - cors.new() + cors_.new() |> origin() - |> cors.allow_method(http.Get) - |> cors.allow_method(http.Post) - |> cors.allow_method(http.Put) - |> cors.allow_method(http.Patch) - |> cors.allow_header("baggage") - |> cors.allow_header("sentry-trace") - |> cors.max_age(86_400) + |> cors_.allow_method(http.Get) + |> cors_.allow_method(http.Post) + |> cors_.allow_method(http.Put) + |> cors_.allow_method(http.Patch) + |> cors_.allow_header("baggage") + |> cors_.allow_header("sentry-trace") + |> cors_.max_age(86_400) } diff --git a/apps/backend/src/tasks/hex.gleam b/apps/backend/src/tasks/hex.gleam index 8241a88..ec6ff24 100644 --- a/apps/backend/src/tasks/hex.gleam +++ b/apps/backend/src/tasks/hex.gleam @@ -82,11 +82,11 @@ fn sync_packages( use all_packages <- result.try(api.get_api_packages_page(page, api_key)) let state = State(..state, newest: first_timestamp(all_packages, state)) let new_packages = take_fresh_packages(all_packages, state.limit) - use state <- result.try(list.try_fold( - new_packages, - state, - do_sync_package(Some(children), force: False), - )) + use state <- result.try({ + list.try_fold(new_packages, state, { + do_sync_package(Some(children), force: False) + }) + }) case list.length(all_packages) == list.length(new_packages) { _ if all_packages == [] -> Ok(state.newest) False -> Ok(state.newest) diff --git a/apps/backend/src/tasks/timeseries.gleam b/apps/backend/src/tasks/timeseries.gleam new file mode 100644 index 0000000..252bed4 --- /dev/null +++ b/apps/backend/src/tasks/timeseries.gleam @@ -0,0 +1,21 @@ +import backend/config.{type Context} +import backend/error +import backend/postgres/queries +import gleam/function +import gleam/list +import gleam/result +import wisp + +pub fn store_timeseries(ctx: Context) { + wisp.log_info("Storing analytics timeseries") + let query = queries.select_last_day_search_analytics(ctx.db) + use analytics <- result.try(query) + use _ <- function.tap({ + result.all({ + use analytic <- list.map(analytics) + queries.upsert_search_analytics_timeseries(ctx.db, analytic) + }) + |> result.map_error(error.debug_log) + }) + wisp.log_info("Storing analytics finished!") +} diff --git a/apps/frontend/gleam.toml b/apps/frontend/gleam.toml index a525a8c..0e45249 100644 --- a/apps/frontend/gleam.toml +++ b/apps/frontend/gleam.toml @@ -12,13 +12,13 @@ typescript_declarations = true gleam_javascript = "~> 0.8" gleam_json = ">= 1.0.1 and < 2.0.0" gleam_stdlib = "~> 0.34 or ~> 1.0" -grille_pain = ">= 1.0.0 and < 2.0.0" lustre_http = "~> 0.5" plinth = ">= 0.2.0 and < 1.0.0" sketch = ">= 2.2.2 and < 3.0.0" lustre = { path = "../../packages/lustre" } vitools = { path = "../../packages/vitools" } modem = ">= 2.0.0 and < 3.0.0" +birl = ">= 1.7.1 and < 2.0.0" [dev-dependencies] gleeunit = "~> 1.0" diff --git a/apps/frontend/manifest.toml b/apps/frontend/manifest.toml index 9e6217f..bf2483b 100644 --- a/apps/frontend/manifest.toml +++ b/apps/frontend/manifest.toml @@ -12,24 +12,22 @@ packages = [ { name = "gleam_otp", version = "0.10.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_stdlib"], otp_app = "gleam_otp", source = "hex", outer_checksum = "0B04FE915ACECE539B317F9652CAADBBC0F000184D586AAAF2D94C100945D72B" }, { name = "gleam_stdlib", version = "0.39.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "2D7DE885A6EA7F1D5015D1698920C9BAF7241102836CE0C3837A4F160128A9C4" }, { name = "gleeunit", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "F7A7228925D3EE7D0813C922E062BFD6D7E9310F0BEE585D3A42F3307E3CFD13" }, - { name = "grille_pain", version = "1.0.1", build_tools = ["gleam"], requirements = ["birl", "gleam_stdlib", "lustre", "plinth", "sketch", "tardis"], otp_app = "grille_pain", source = "hex", outer_checksum = "F0CA9AA0BD4D03B8E190AB4CBB9429DE9389BC2152CF566C3410261F5729827C" }, { name = "lustre", version = "4.3.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_json", "gleam_otp", "gleam_stdlib"], source = "local", path = "../../packages/lustre" }, { name = "lustre_http", version = "0.5.2", build_tools = ["gleam"], requirements = ["gleam_fetch", "gleam_http", "gleam_javascript", "gleam_json", "gleam_stdlib", "lustre"], otp_app = "lustre_http", source = "hex", outer_checksum = "FB0478CBFA6B16DBE8ECA326DAE2EC15645E04900595EF2C4F039ABFA0512ABA" }, { name = "modem", version = "2.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib", "lustre"], otp_app = "modem", source = "hex", outer_checksum = "FF07984B15D553B15A45722F64BC440CA22B6139930E95E16EB956F1E24886AD" }, { name = "plinth", version = "0.4.9", build_tools = ["gleam"], requirements = ["conversation", "gleam_javascript", "gleam_json", "gleam_stdlib"], otp_app = "plinth", source = "hex", outer_checksum = "663C788C00FF908662B47B78B1CEBE1260AB814B45531AA42EBAEE974CDC7E27" }, { name = "ranger", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "ranger", source = "hex", outer_checksum = "1566C272B1D141B3BBA38B25CB761EF56E312E79EC0E2DFD4D3C19FB0CC1F98C" }, { name = "sketch", version = "2.3.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_otp", "gleam_stdlib", "lustre", "plinth"], otp_app = "sketch", source = "hex", outer_checksum = "C4BDDB207A3CC94AC83BFB4D1300C51A9D0048569213ED4B35C9D982B305AF9D" }, - { name = "tardis", version = "0.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib", "lustre", "plinth", "sketch"], otp_app = "tardis", source = "hex", outer_checksum = "C8E7BAB95C59EF50332905A06B82BB35526B7BE2F191580F3CD8790903AA49F7" }, { name = "thoas", version = "1.2.1", build_tools = ["rebar3"], requirements = [], otp_app = "thoas", source = "hex", outer_checksum = "E38697EDFFD6E91BD12CEA41B155115282630075C2A727E7A6B2947F5408B86A" }, { name = "vitools", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], source = "local", path = "../../packages/vitools" }, ] [requirements] +birl = { version = ">= 1.7.1 and < 2.0.0" } gleam_javascript = { version = "~> 0.8" } gleam_json = { version = ">= 1.0.1 and < 2.0.0" } gleam_stdlib = { version = "~> 0.34 or ~> 1.0" } gleeunit = { version = "~> 1.0" } -grille_pain = { version = ">= 1.0.0 and < 2.0.0" } lustre = { path = "../../packages/lustre" } lustre_http = { version = "~> 0.5" } modem = { version = ">= 2.0.0 and < 3.0.0" } diff --git a/apps/frontend/package.json b/apps/frontend/package.json index 154f7cf..e9c6a5f 100644 --- a/apps/frontend/package.json +++ b/apps/frontend/package.json @@ -9,9 +9,10 @@ "frontend:preview": "vite preview" }, "dependencies": { - "@chouqueth/gleam": "^1.3.2", + "@chouqueth/gleam": "^1.4.1", "@gleam-lang/highlight.js-gleam": "^1.5.0", "@sentry/browser": "^8.0.0", + "chart.js": "^4.4.4", "dompurify": "^3.1.4", "highlight.js": "^11.9.0", "marked": "^12.0.2", diff --git a/apps/frontend/src/chart.mjs b/apps/frontend/src/chart.mjs new file mode 100644 index 0000000..01d1f6d --- /dev/null +++ b/apps/frontend/src/chart.mjs @@ -0,0 +1,69 @@ +import { Chart } from 'chart.js/auto' + +export class LineChart extends HTMLElement { + static observedAttributes = ['datasets'] + + #shadow + #canvas + dataset + + constructor() { + super() + this.#shadow = this.attachShadow({ mode: 'open' }) + } + + connectedCallback() { + this.#render() + } + + #render() { + const labels = this.datasets.labels.toArray() + const data = this.datasets.data.toArray() + const wrapper = document.createElement('div') + wrapper.style.position = 'relative' + // wrapper.style.maxWidth = '400px' + // wrapper.style.maxHeight = '150px' + this.#canvas = document.createElement('canvas') + wrapper.appendChild(this.#canvas) + this.#shadow.appendChild(wrapper) + + new Chart(this.#canvas, { + type: 'line', + data: { + labels, + datasets: [ + { + data, + borderColor: '#8c3a96', + fill: false, + tension: 0.4, + }, + ], + }, + options: { + responsive: true, + animation: false, + events: [], + plugins: { + legend: { + display: false, + }, + }, + scales: { + x: { display: true, title: { display: true } }, + y: { display: true, title: { display: true, text: 'Value' } }, + }, + }, + }) + } + + // Lifecycle functions. + disconnectedCallback() {} + adoptedCallback() {} + + attributeChangedCallback() {} + + static register() { + customElements.define('line-chart', LineChart) + } +} diff --git a/apps/frontend/src/data/model.gleam b/apps/frontend/src/data/model.gleam index 59ee596..65cbdda 100644 --- a/apps/frontend/src/data/model.gleam +++ b/apps/frontend/src/data/model.gleam @@ -1,3 +1,4 @@ +import birl import data/kind import data/msg.{type Msg} import data/package.{type Package} @@ -36,6 +37,10 @@ pub type Model { show_old_packages: Bool, show_documentation_search: Bool, show_vector_search: Bool, + total_searches: Int, + total_signatures: Int, + total_packages: Int, + timeseries: List(#(Int, birl.Time)), ) } @@ -62,6 +67,10 @@ pub fn init() { show_old_packages: False, show_documentation_search: False, show_vector_search: False, + total_searches: 0, + total_signatures: 0, + total_packages: 0, + timeseries: [], ) } @@ -93,6 +102,21 @@ pub fn update_input(model: Model, content: String) { Model(..model, input: content) } +pub fn update_analytics( + model: Model, + analytics: #(Int, Int, Int, List(#(Int, birl.Time))), +) { + let #(total_searches, total_signatures, total_packages, timeseries) = + analytics + Model( + ..model, + timeseries:, + total_searches:, + total_signatures:, + total_packages:, + ) +} + pub fn search_key(key key: String, model model: Model) { key <> string.inspect([ @@ -324,6 +348,10 @@ pub fn reset(model: Model) { show_old_packages: False, show_documentation_search: False, show_vector_search: False, + timeseries: [], + total_searches: 0, + total_signatures: 0, + total_packages: 0, ) } diff --git a/apps/frontend/src/data/msg.gleam b/apps/frontend/src/data/msg.gleam index 32780a2..0918e00 100644 --- a/apps/frontend/src/data/msg.gleam +++ b/apps/frontend/src/data/msg.gleam @@ -1,3 +1,4 @@ +import birl import data/package import data/search_result.{type SearchResults} import frontend/router @@ -25,6 +26,7 @@ pub type Msg { Reset ScrollTo(String) OnEscape + Analytics(Result(#(Int, Int, Int, List(#(Int, birl.Time))), http.HttpError)) OnRouteChange(router.Route) OnCheckFilter(Filter, Bool) } diff --git a/apps/frontend/src/frontend.gleam b/apps/frontend/src/frontend.gleam index f4539da..2e1e5cd 100644 --- a/apps/frontend/src/frontend.gleam +++ b/apps/frontend/src/frontend.gleam @@ -1,3 +1,4 @@ +import birl import data/model.{type Model} import data/msg.{type Msg} import data/package @@ -12,11 +13,13 @@ import gleam/bool import gleam/dict import gleam/dynamic import gleam/option.{None, Some} +import gleam/pair import gleam/result import gleam/uri.{type Uri} -import grille_pain -import grille_pain/lustre/toast -import grille_pain/options + +// import grille_pain +// import grille_pain/lustre/toast +// import grille_pain/options import lustre import lustre/effect import lustre/lazy @@ -59,10 +62,10 @@ pub fn main() { let assert Ok(cache) = sketch.setup(sketch_options.node()) let assert Ok(_) = lazy.setup() let assert Ok(_) = sr.setup() - let assert Ok(_) = - options.default() - |> options.timeout(5000) - |> grille_pain.setup() + // let assert Ok(_) = + // options.default() + // |> options.timeout(5000) + // |> grille_pain.setup() let assert Ok(_) = view.view @@ -86,6 +89,32 @@ fn init(_) { http.expect_json(dynamic.list(package.decoder), msg.Trendings) |> http.get(config.api_endpoint() <> "/trendings", _), ) + |> update.add_effect( + msg.Analytics + |> http.expect_json( + dynamic.decode4( + fn(a, b, c, d) { #(a, b, c, d) }, + dynamic.field("total", dynamic.int), + dynamic.field("signatures", dynamic.int), + dynamic.field("packages", dynamic.int), + dynamic.field("timeseries", { + dynamic.list(dynamic.decode2( + pair.new, + dynamic.field("count", dynamic.int), + dynamic.field("date", fn(dyn) { + dynamic.string(dyn) + |> result.then(fn(t) { + birl.parse(t) + |> result.replace_error([]) + }) + }), + )) + }), + ), + _, + ) + |> http.get(config.api_endpoint() <> "/analytics", _), + ) } fn on_url_change(uri: Uri) -> Msg { @@ -113,6 +142,15 @@ fn update(model: Model, msg: Msg) { handle_search_results(model, input, search_results) msg.OnCheckFilter(filter, value) -> handle_oncheck_filter(model, filter, value) + msg.Analytics(analytics) -> { + case analytics { + Error(_) -> #(model, effect.none()) + Ok(analytics) -> + model + |> model.update_analytics(analytics) + |> update.none() + } + } } } @@ -193,6 +231,7 @@ fn handle_route_change(model: Model, route: router.Route) { case route { router.Home -> model.update_input(model, "") router.Trending -> model.update_input(model, "") + router.Analytics -> model.update_input(model, "") router.Search(q) -> model.update_input(model, q) |> model.update_submitted_input @@ -207,7 +246,8 @@ fn display_toast( |> result.map_error(fn(error) { toast_error.describe_http_error(error) |> option.map(errors.capture_message) - |> option.map(toast.error) + // |> option.map(toast.error) + |> option.map(fn(_) { effect.none() }) }) |> result.unwrap_error(option.None) |> option.unwrap(effect.none()) diff --git a/apps/frontend/src/frontend.ts b/apps/frontend/src/frontend.ts index 4501e9f..0145d33 100644 --- a/apps/frontend/src/frontend.ts +++ b/apps/frontend/src/frontend.ts @@ -3,12 +3,15 @@ import gleamHljs from '@gleam-lang/highlight.js-gleam' import hljs from 'highlight.js/lib/core' import plaintext from 'highlight.js/lib/languages/plaintext' // @ts-ignore +import { LineChart } from './chart.mjs' +// @ts-ignore import { main } from './frontend.gleam' import './stylesheets/all.css' import './stylesheets/hljs-theme.css' import './stylesheets/main.css' import './stylesheets/normalize.css' +LineChart.register() // @ts-ignore Element.prototype._attachShadow = Element.prototype.attachShadow Element.prototype.attachShadow = function () { diff --git a/apps/frontend/src/frontend/router.gleam b/apps/frontend/src/frontend/router.gleam index 00080fd..1525a0a 100644 --- a/apps/frontend/src/frontend/router.gleam +++ b/apps/frontend/src/frontend/router.gleam @@ -9,12 +9,14 @@ pub type Route { Home Search(query: String) Trending + Analytics } pub fn parse_uri(uri: Uri) -> Route { case uri.path_segments(uri.path) { ["search"] -> handle_search_path(uri) ["trending"] -> Trending + ["analytics"] -> Analytics _ -> Home } } @@ -38,5 +40,6 @@ pub fn update_page_title(route: Route) { Home -> ffi.update_title("Gloogle") Search(q) -> ffi.update_title("Gloogle — Search " <> q) Trending -> ffi.update_title("Gloogle — Trending") + Analytics -> ffi.update_title("Gloogle — Analytics") } } diff --git a/apps/frontend/src/frontend/view/body/body.gleam b/apps/frontend/src/frontend/view/body/body.gleam index fd8ee91..4dd38a1 100644 --- a/apps/frontend/src/frontend/view/body/body.gleam +++ b/apps/frontend/src/frontend/view/body/body.gleam @@ -1,3 +1,4 @@ +import birl import data/model.{type Model} import data/msg import data/search_result @@ -8,8 +9,13 @@ import frontend/strings as frontend_strings import frontend/view/search_input/search_input import gleam/bool import gleam/dict +import gleam/float +import gleam/int +import gleam/io +import gleam/list import gleam/result import gleam/string +import line_chart import lustre/attribute as a import lustre/element as el import lustre/element/html as h @@ -167,10 +173,7 @@ fn sidebar(model: Model) { ]), h.div([a.class("sidebar-spacer")], []), h.div([a.class("sidebar-links")], [ - // s.sidebar_link_wrapper([], [ - // s.sidebar_icon([], [icons.trends()]), - // s.sidebar_link([], [el.text("Trends")]), - // ]), + sidebar_link(href: "/analytics", icon: icons.trends(), title: "Analytics"), // s.sidebar_link_wrapper([], [ // s.sidebar_icon([], [icons.shortcuts()]), // s.sidebar_link([], [el.text("Shortcuts")]), @@ -179,7 +182,7 @@ fn sidebar(model: Model) { // s.sidebar_icon([], [icons.gift()]), // s.sidebar_link([], [el.text("Resources")]), // ]), - sidebar_link( + sidebar_external_link( href: "https://github.com/sponsors/ghivert", title: "Sponsor", icon: icons.heart(), @@ -207,7 +210,7 @@ fn checkbox(active: Bool, msg: msg.Filter, name: String) { ]) } -fn sidebar_link(href href: String, title title: String, icon icon) { +fn sidebar_external_link(href href: String, title title: String, icon icon) { let class = a.class("sidebar-link-wrapper") h.a([class, a.href(href), a.target("_blank"), a.rel("noreferrer noopener")], [ h.div([a.class("sidebar-icon")], [icon]), @@ -215,10 +218,85 @@ fn sidebar_link(href href: String, title title: String, icon icon) { ]) } +fn sidebar_link(href href: String, title title: String, icon icon) { + let class = a.class("sidebar-link-wrapper") + h.a([class, a.href(href)], [ + h.div([a.class("sidebar-icon")], [icon]), + h.div([a.class("sidebar-link")], [el.text(title)]), + ]) +} + +fn format_huge_number(number: Int) { + let number = int.to_float(number) + let g = number /. 1_000_000_000.0 + let m = number /. 1_000_000.0 + let k = number /. 1000.0 + case number { + _ if g >. 1.0 -> float.to_string(g) |> string.slice(0, 5) <> " G" + _ if m >. 1.0 -> float.to_string(m) |> string.slice(0, 5) <> " M" + _ if k >. 1.0 -> float.to_string(k) |> string.slice(0, 5) <> " K" + _ -> float.round(number) |> int.to_string + } +} + pub fn body(model: Model) { case model.route { router.Home -> h.main([a.class("main")], [view_search_input(model)]) router.Trending -> h.main([a.class("main")], [view_trending(model)]) + router.Analytics -> + el.fragment([ + sidebar(model), + h.main([a.class("main")], [ + h.div( + [a.class("items-wrapper"), a.style([#("padding-left", "24px")])], + [ + h.div([a.class("matches-titles")], [ + h.div([a.class("matches-title")], [h.text("Global analytics")]), + ]), + h.div([a.class("analytics-box-wrapper")], [ + h.div([a.class("analytics-box")], [ + h.div([a.class("analytics-title")], [ + h.text("Number of searches"), + ]), + h.text(format_huge_number(model.total_searches)), + ]), + h.div([a.class("analytics-box")], [ + h.div([a.class("analytics-title")], [ + h.text("Number of signatures indexed"), + ]), + h.text(format_huge_number(model.total_signatures)), + ]), + h.div([a.class("analytics-box")], [ + h.div([a.class("analytics-title")], [ + h.text("Number of packages indexed"), + ]), + h.text(format_huge_number(model.total_packages)), + ]), + ]), + h.div([a.class("matches-titles")], [ + h.div([a.class("matches-title")], [h.text("Last 30 days")]), + ]), + h.div([a.style([#("width", "auto"), #("height", "500px")])], [ + case model.timeseries { + [] -> el.none() + data -> { + line_chart.line_chart({ + use line_chart.Dataset(dates, value), #(count, date) <- list.fold( + data, + line_chart.Dataset([], []), + ) + line_chart.Dataset([birl.to_iso8601(date), ..dates], [ + count, + ..value + ]) + }) + } + }, + ]), + ], + ), + ]), + ]) router.Search(_) -> { let key = model.search_key(model.submitted_input, model) el.fragment([ diff --git a/apps/frontend/src/frontend/view/navbar/navbar.gleam b/apps/frontend/src/frontend/view/navbar/navbar.gleam index 712a23f..ca00942 100644 --- a/apps/frontend/src/frontend/view/navbar/navbar.gleam +++ b/apps/frontend/src/frontend/view/navbar/navbar.gleam @@ -22,7 +22,7 @@ pub fn navbar(model: Model) { s.navbar(transparent, [a.class("navbar")], [ case model.route { router.Home -> h.div([], []) - router.Search(_) | router.Trending -> + router.Search(_) | router.Trending | router.Analytics -> s.navbar_search([], [ s.navbar_search_title([a.href("/")], [ s.search_lucy(40, [a.src("/images/lucy.svg")]), diff --git a/apps/frontend/src/line_chart.gleam b/apps/frontend/src/line_chart.gleam new file mode 100644 index 0000000..b2943b3 --- /dev/null +++ b/apps/frontend/src/line_chart.gleam @@ -0,0 +1,16 @@ +import gleam/string +import lustre/attribute +import lustre/element + +pub type Dataset { + Dataset(labels: List(String), data: List(Int)) +} + +pub fn line_chart(datasets: Dataset) { + let datasets = attribute.property("datasets", datasets) + element.element( + "line-chart", + [attribute.style([#("display", "block")]), datasets], + [], + ) +} diff --git a/apps/frontend/src/stylesheets/all.css b/apps/frontend/src/stylesheets/all.css index cf31680..10d2d80 100644 --- a/apps/frontend/src/stylesheets/all.css +++ b/apps/frontend/src/stylesheets/all.css @@ -553,3 +553,26 @@ lazy-node:has(:not(:defined)) { line-height: 1.6; font-weight: 400; } + +.analytics-box { + display: flex; + flex-direction: column; + gap: 24px; + padding: 24px; + border: 1px solid var(--border-color); + border-radius: 10px; + font-size: 2.5rem; + line-height: 1.75; + width: 200px; + /* height: 200px; */ + justify-content: space-between; +} + +.analytics-title { + font-size: 1rem; +} + +.analytics-box-wrapper { + display: flex; + gap: 24px; +} diff --git a/yarn.lock b/yarn.lock index 692f571..7c816ea 100644 --- a/yarn.lock +++ b/yarn.lock @@ -304,6 +304,18 @@ __metadata: languageName: node linkType: hard +"@chouqueth/gleam@npm:^1.4.1": + version: 1.4.1 + resolution: "@chouqueth/gleam@npm:1.4.1" + dependencies: + cachedir: "npm:^2.4.0" + tar: "npm:^7.1.0" + bin: + gleam: bin/cli.mjs + checksum: 10c0/d7238a489c699ff1fdab1701603e3066132826eb005926b1e3f2ed4fae0a894c34016cf8b32ac88495c2359a6ec4d447697a09fe3680ddd6b441611b4f42efb4 + languageName: node + linkType: hard + "@esbuild/aix-ppc64@npm:0.20.2": version: 0.20.2 resolution: "@esbuild/aix-ppc64@npm:0.20.2" @@ -868,6 +880,13 @@ __metadata: languageName: node linkType: hard +"@kurkle/color@npm:^0.3.0": + version: 0.3.2 + resolution: "@kurkle/color@npm:0.3.2" + checksum: 10c0/a9e8e3e35dcd59dec4dd4f0105919c05e24823a96347bcf152965c29e48d6290b66d5fb96c071875db752e10930724c48ce6d338fefbd65e0ce5082d5c78970e + languageName: node + linkType: hard + "@npmcli/agent@npm:^2.0.0": version: 2.2.2 resolution: "@npmcli/agent@npm:2.2.2" @@ -1631,6 +1650,15 @@ __metadata: languageName: node linkType: hard +"chart.js@npm:^4.4.4": + version: 4.4.4 + resolution: "chart.js@npm:4.4.4" + dependencies: + "@kurkle/color": "npm:^0.3.0" + checksum: 10c0/9fa3206403a6103916f7762c2665d322c42b0cc07fba91526b1d033ddb887c1ba74b3ebc0bd0748a9e55abd1017f25fdb2292cdd6579d8c2d3bcb1c58f71281c + languageName: node + linkType: hard + "check-error@npm:^1.0.3": version: 1.0.3 resolution: "check-error@npm:1.0.3" @@ -2453,12 +2481,13 @@ __metadata: version: 0.0.0-use.local resolution: "frontend@workspace:apps/frontend" dependencies: - "@chouqueth/gleam": "npm:^1.3.2" + "@chouqueth/gleam": "npm:^1.4.1" "@gleam-lang/highlight.js-gleam": "npm:^1.5.0" "@sentry/browser": "npm:^8.0.0" "@sentry/vite-plugin": "npm:^2.16.1" "@trivago/prettier-plugin-sort-imports": "npm:^4.3.0" "@types/dompurify": "npm:^3.0.5" + chart.js: "npm:^4.4.4" dompurify: "npm:^3.1.4" dotenv: "npm:^16.4.5" highlight.js: "npm:^11.9.0"