Fix/add timeseries

.github/workflows/test.yml

@@ -17,7 +17,7 @@ jobs:
  uses: erlef/setup-beam@v1
  with:
  otp-version: '26.0.2'
- gleam-version: '1.2.1'
+ gleam-version: '1.4.1'
  rebar3-version: '3'
  # elixir-version: "1.15.4"
  - name: Download gleam dependencies
@@ -51,7 +51,7 @@ jobs:
  uses: erlef/setup-beam@v1
  with:
  otp-version: '26.0.2'
- gleam-version: '1.2.1'
+ gleam-version: '1.4.1'
  rebar3-version: '3'
  - name: Download gleam dependencies
  run: gleam deps download

.mise.toml

@@ -1,6 +1,6 @@
 # Runtimes used by the stack.
 [tools]
-gleam = "1.4.0"
+gleam = "1.4.1"
 erlang = "27"
 node = "22"
 deno = "latest"

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=x86_64 ghcr.io/gleam-lang/gleam:v1.4.0-erlang-alpine AS builder
+FROM --platform=x86_64 ghcr.io/gleam-lang/gleam:v1.4.1-erlang-alpine AS builder
 
 RUN apk add build-base ca-certificates
 RUN mkdir -p /build/backend/src

apps/backend/db/migrations/20240902224247_create_analytics_timeseries.sql

@@ -0,0 +1,10 @@
+-- migrate:up
+create table analytics_timeseries (
+ query text not null,
+ occurences int not null,
+ date timestamptz not null,
+ primary key (query, date)
+);
+
+-- migrate:down
+drop table analytics_timeseries;

apps/backend/db/migrations/20240902225236_alter_default_value_in_search_analytics.sql

@@ -0,0 +1,7 @@
+-- migrate:up
+alter table only search_analytics alter column occurences set default 1;
+update search_analytics set occurences = occurences + 1;
+
+-- migrate:down
+alter table only search_analytics alter column occurences set default 0;
+update search_analytics set occurences = occurences - 1;

apps/backend/db/schema.sql

@@ -96,6 +96,17 @@ ALTER TABLE public.analytics ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY (
 );
 
 
+--
+-- Name: analytics_timeseries; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.analytics_timeseries (
+ query text NOT NULL,
+ occurences integer NOT NULL,
+ date timestamp with time zone NOT NULL
+);
+
+
 --
 -- Name: hex_read; Type: TABLE; Schema: public; Owner: -
 --
@@ -304,7 +315,7 @@ CREATE TABLE public.schema_migrations (
 
 CREATE TABLE public.search_analytics (
  query text NOT NULL,
- occurences integer DEFAULT 0 NOT NULL,
+ occurences integer DEFAULT 1 NOT NULL,
  created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
  updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL
 );
@@ -326,6 +337,14 @@ ALTER TABLE ONLY public.analytics
  ADD CONSTRAINT analytics_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: analytics_timeseries analytics_timeseries_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.analytics_timeseries
+ ADD CONSTRAINT analytics_timeseries_pkey PRIMARY KEY (query, date);
+
+
 --
 -- Name: hex_read hex_read_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -590,4 +609,6 @@ INSERT INTO public.schema_migrations (version) VALUES
  ('20240521204341'),
  ('20240801164720'),
  ('20240801211520'),
- ('20240801220817');
+ ('20240801220817'),
+ ('20240902224247'),
+ ('20240902225236');

apps/backend/gleam.toml

@@ -4,7 +4,9 @@ version = "1.0.0"
 [dependencies]
 aws4_request = ">= 0.1.1 and < 1.0.0"
 birl = "~> 1.6"
-dot_env = "~> 0.5"
+chomp = ">= 0.1.0 and < 1.0.0"
+cors_builder = ">= 2.0.0 and < 3.0.0"
+decipher = ">= 1.2.0 and < 2.0.0"
 gleam_erlang = "~> 0.25"
 gleam_hexpm = "~> 1.0"
 gleam_http = "~> 3.6"
@@ -14,19 +16,16 @@ gleam_otp = "~> 0.10"
 gleam_package_interface = ">= 1.0.0 and < 2.0.0"
 gleam_pgo = {path = "../../packages/pgo"}
 gleam_stdlib = "~> 0.34 or ~> 1.0"
-mist = ">= 1.0.0 and < 2.0.0"
+glexer = ">= 1.0.1 and < 2.0.0"
+mist = ">= 1.2.0 and < 2.0.0"
 pgo = "~> 0.14"
 prng = ">= 3.0.3 and < 4.0.0"
 radiate = ">= 0.4.0 and < 1.0.0"
 ranger = ">= 1.2.0 and < 2.0.0"
-simplifile = ">= 1.7.0 and < 2.0.0"
+simplifile = ">= 2.1.0 and < 3.0.0"
 tom = { path ="../../packages/tom" }
 verl = ">= 1.1.1 and < 2.0.0"
-wisp = "~> 0.14"
-cors_builder = ">= 1.0.0 and < 2.0.0"
-decipher = ">= 1.2.0 and < 2.0.0"
-chomp = ">= 0.1.0 and < 1.0.0"
-glexer = ">= 1.0.1 and < 2.0.0"
+wisp = ">= 1.1.0 and < 2.0.0"
 
 [dev-dependencies]
 gleeunit = "~> 1.0"

apps/backend/manifest.toml

@@ -6,9 +6,10 @@ packages = [
  { name = "backoff", version = "1.1.6", build_tools = ["rebar3"], requirements = [], otp_app = "backoff", source = "hex", outer_checksum = "CF0CFFF8995FB20562F822E5CC47D8CCF664C5ECDC26A684CBE85C225F9D7C39" },
  { name = "birl", version = "1.7.1", build_tools = ["gleam"], requirements = ["gleam_stdlib", "ranger"], otp_app = "birl", source = "hex", outer_checksum = "5C66647D62BCB11FE327E7A6024907C4A17954EF22865FE0940B54A852446D01" },
  { name = "chomp", version = "0.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "chomp", source = "hex", outer_checksum = "C87304897B4D4DEA69420DB2FF88B087673AAE9EC09CA8A0FBF4675F605767C2" },
- { name = "cors_builder", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_stdlib", "mist", "wisp"], otp_app = "cors_builder", source = "hex", outer_checksum = "951B5B648E958BD6181A6EED98BCA4EEB302B83DC7DCE2954B3462114209EC43" },
+ { name = "cors_builder", version = "2.0.0", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_stdlib", "mist", "wisp"], otp_app = "cors_builder", source = "hex", outer_checksum = "E4634D1CAD89BC26B4FE986B45D55268BA5369E9224C2FD7872C0DB5AC028E47" },
  { name = "decipher", version = "1.2.0", build_tools = ["gleam"], requirements = ["birl", "gleam_json", "gleam_stdlib", "stoiridh_version"], otp_app = "decipher", source = "hex", outer_checksum = "9F1B5C6FF0D798046E4E0EF87D09DD729324CB72BD7F0D4152B797324D51223E" },
- { name = "dot_env", version = "0.5.1", build_tools = ["gleam"], requirements = ["gleam_stdlib", "simplifile"], otp_app = "dot_env", source = "hex", outer_checksum = "AF5C972D6129F67AF3BB00134AB2808D37111A8D61686CFA86F3ADF652548982" },
+ { name = "directories", version = "1.1.0", build_tools = ["gleam"], requirements = ["envoy", "gleam_stdlib", "platform", "simplifile"], otp_app = "directories", source = "hex", outer_checksum = "BDA521A4EB9EE3A7894F0DC863797878E91FF5C7826F7084B2E731E208BDB076" },
+ { name = "envoy", version = "1.0.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "envoy", source = "hex", outer_checksum = "CFAACCCFC47654F7E8B75E614746ED924C65BD08B1DE21101548AC314A8B6A41" },
  { name = "exception", version = "2.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "exception", source = "hex", outer_checksum = "F5580D584F16A20B7FCDCABF9E9BE9A2C1F6AC4F9176FA6DD0B63E3B20D450AA" },
  { name = "filepath", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "EFB6FF65C98B2A16378ABC3EE2B14124168C0CE5201553DE652E2644DCFDB594" },
  { name = "filespy", version = "0.5.0", build_tools = ["gleam"], requirements = ["fs", "gleam_erlang", "gleam_otp", "gleam_stdlib"], otp_app = "filespy", source = "hex", outer_checksum = "F8E7A9C9CA86D68CCC25491125BFF36BEF7483892D7BEC24AA30D6B540504F06" },
@@ -30,33 +31,33 @@ packages = [
  { name = "glisten", version = "2.0.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_otp", "gleam_stdlib"], otp_app = "glisten", source = "hex", outer_checksum = "CF3A9383E9BA4A8CBAF2F7B799716290D02F2AC34E7A77556B49376B662B9314" },
  { name = "gramps", version = "2.0.3", build_tools = ["gleam"], requirements = ["gleam_crypto", "gleam_erlang", "gleam_http", "gleam_stdlib"], otp_app = "gramps", source = "hex", outer_checksum = "3CCAA6E081225180D95C79679D383BBF51C8D1FDC1B84DA1DA444F628C373793" },
  { name = "hpack_erl", version = "0.3.0", build_tools = ["rebar3"], requirements = [], otp_app = "hpack", source = "hex", outer_checksum = "D6137D7079169D8C485C6962DFE261AF5B9EF60FBC557344511C1E65E3D95FB0" },
- { name = "logging", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "logging", source = "hex", outer_checksum = "FCB111401BDB4703A440A94FF8CC7DA521112269C065F219C2766998333E7738" },
+ { name = "logging", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "logging", source = "hex", outer_checksum = "1098FBF10B54B44C2C7FDF0B01C1253CAFACDACABEFB4B0D027803246753E06D" },
  { name = "marceau", version = "1.2.0", build_tools = ["gleam"], requirements = [], otp_app = "marceau", source = "hex", outer_checksum = "5188D643C181EE350D8A20A3BDBD63AF7B6C505DE333CFBE05EF642ADD88A59B" },
  { name = "mist", version = "1.2.0", build_tools = ["gleam"], requirements = ["birl", "gleam_erlang", "gleam_http", "gleam_otp", "gleam_stdlib", "glisten", "gramps", "hpack_erl", "logging"], otp_app = "mist", source = "hex", outer_checksum = "109B4D64E68C104CC23BB3CC5441ECD479DD7444889DA01113B75C6AF0F0E17B" },
  { name = "opentelemetry_api", version = "1.3.0", build_tools = ["rebar3", "mix"], requirements = ["opentelemetry_semantic_conventions"], otp_app = "opentelemetry_api", source = "hex", outer_checksum = "B9E5FF775FD064FA098DBA3C398490B77649A352B40B0B730A6B7DC0BDD68858" },
  { name = "opentelemetry_semantic_conventions", version = "0.2.0", build_tools = ["rebar3", "mix"], requirements = [], otp_app = "opentelemetry_semantic_conventions", source = "hex", outer_checksum = "D61FA1F5639EE8668D74B527E6806E0503EFC55A42DB7B5F39939D84C07D6895" },
  { name = "pg_types", version = "0.4.0", build_tools = ["rebar3"], requirements = [], otp_app = "pg_types", source = "hex", outer_checksum = "B02EFA785CAECECF9702C681C80A9CA12A39F9161A846CE17B01FB20AEEED7EB" },
  { name = "pgo", version = "0.14.0", build_tools = ["rebar3"], requirements = ["backoff", "opentelemetry_api", "pg_types"], otp_app = "pgo", source = "hex", outer_checksum = "71016C22599936E042DC0012EE4589D24C71427D266292F775EBF201D97DF9C9" },
+ { name = "platform", version = "1.0.0", build_tools = ["gleam"], requirements = [], otp_app = "platform", source = "hex", outer_checksum = "8339420A95AD89AAC0F82F4C3DB8DD401041742D6C3F46132A8739F6AEB75391" },
  { name = "pprint", version = "1.0.3", build_tools = ["gleam"], requirements = ["glam", "gleam_stdlib"], otp_app = "pprint", source = "hex", outer_checksum = "76BBB92E23D12D954BD452686543F29EDE8EBEBB7FC0ACCBCA66EEF276EC3A06" },
  { name = "prng", version = "3.0.3", build_tools = ["gleam"], requirements = ["gleam_bitwise", "gleam_stdlib"], otp_app = "prng", source = "hex", outer_checksum = "53006736FE23A0F61828C95B505193E10905D8DB76E128F1642D3E571E08F589" },
  { name = "radiate", version = "0.4.0", build_tools = ["gleam"], requirements = ["filespy", "gleam_otp", "gleam_stdlib", "shellout"], otp_app = "radiate", source = "hex", outer_checksum = "93A76A66EE4741DBFD3E79E27CBD11FE58EC3CB1C58F017FC165944E339D6293" },
  { name = "ranger", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "ranger", source = "hex", outer_checksum = "1566C272B1D141B3BBA38B25CB761EF56E312E79EC0E2DFD4D3C19FB0CC1F98C" },
  { name = "shellout", version = "1.6.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "shellout", source = "hex", outer_checksum = "E2FCD18957F0E9F67E1F497FC9FF57393392F8A9BAEAEA4779541DE7A68DD7E0" },
- { name = "simplifile", version = "1.7.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "1D5DFA3A2F9319EC85825F6ED88B8E449F381B0D55A62F5E61424E748E7DDEB0" },
+ { name = "simplifile", version = "2.1.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "BDD04F5D31D6D34E2EDFAEF0B68A6297AEC939888C3BFCE61133DE13857F6DA2" },
  { name = "stoiridh_version", version = "0.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "stoiridh_version", source = "hex", outer_checksum = "EEF8ADAB9755BD33EB202F169376F1A7797AEF90823FDCA671D8590D04FBF56B" },
  { name = "thoas", version = "1.2.1", build_tools = ["rebar3"], requirements = [], otp_app = "thoas", source = "hex", outer_checksum = "E38697EDFFD6E91BD12CEA41B155115282630075C2A727E7A6B2947F5408B86A" },
  { name = "tom", version = "0.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], source = "local", path = "../../packages/tom" },
  { name = "verl", version = "1.1.1", build_tools = ["rebar3"], requirements = [], otp_app = "verl", source = "hex", outer_checksum = "0925E51CD92A0A8BE271765B02430B2E2CFF8AC30EF24D123BD0D58511E8FB18" },
- { name = "wisp", version = "0.14.0", build_tools = ["gleam"], requirements = ["exception", "gleam_crypto", "gleam_erlang", "gleam_http", "gleam_json", "gleam_stdlib", "logging", "marceau", "mist", "simplifile"], otp_app = "wisp", source = "hex", outer_checksum = "9F5453AF1F9275E6F8707BC815D6A6A9DF41551921B16FBDBA52883773BAE684" },
+ { name = "wisp", version = "1.1.0", build_tools = ["gleam"], requirements = ["directories", "exception", "gleam_crypto", "gleam_erlang", "gleam_http", "gleam_json", "gleam_stdlib", "logging", "marceau", "mist", "simplifile"], otp_app = "wisp", source = "hex", outer_checksum = "5B25E37E08A94039668D2ACF7B2D2A665BEAF4B48EFA8613010B4E3164137664" },
 ]
 
 [requirements]
 aws4_request = { version = ">= 0.1.1 and < 1.0.0" }
 birl = { version = "~> 1.6" }
 chomp = { version = ">= 0.1.0 and < 1.0.0" }
-cors_builder = { version = ">= 1.0.0 and < 2.0.0" }
+cors_builder = { version = ">= 2.0.0 and < 3.0.0" }
 decipher = { version = ">= 1.2.0 and < 2.0.0" }
-dot_env = { version = "~> 0.5" }
 gleam_erlang = { version = "~> 0.25" }
 gleam_hexpm = { version = "~> 1.0" }
 gleam_http = { version = "~> 3.6" }
@@ -68,13 +69,13 @@ gleam_pgo = { path = "../../packages/pgo" }
 gleam_stdlib = { version = "~> 0.34 or ~> 1.0" }
 gleeunit = { version = "~> 1.0" }
 glexer = { version = ">= 1.0.1 and < 2.0.0" }
-mist = { version = ">= 1.0.0 and < 2.0.0" }
+mist = { version = ">= 1.2.0 and < 2.0.0" }
 pgo = { version = "~> 0.14" }
 pprint = { version = ">= 1.0.3 and < 2.0.0" }
 prng = { version = ">= 3.0.3 and < 4.0.0" }
 radiate = { version = ">= 0.4.0 and < 1.0.0" }
 ranger = { version = ">= 1.2.0 and < 2.0.0" }
-simplifile = { version = ">= 1.7.0 and < 2.0.0" }
+simplifile = { version = ">= 2.1.0 and < 3.0.0" }
 tom = { path = "../../packages/tom" }
 verl = { version = ">= 1.1.1 and < 2.0.0" }
-wisp = { version = "~> 0.14" }
+wisp = { version = ">= 1.1.0 and < 2.0.0" }

apps/backend/src/api/github.gleam

@@ -25,7 +25,7 @@ fn query(
  use response <- result.try(
  request.new()
  |> request.set_header("authorization", "Bearer " <> token)
- |> request.set_header("user-agent", "gloogle / 0.0.0")
+ |> request.set_header("user-agent", "gloogle / 1.0.0")
  |> request.set_method(http.Post)
  |> request.set_scheme(http.Https)
  |> request.set_host("api.github.com")

apps/backend/src/api/github/stargazer_count.gleam

@@ -15,7 +15,7 @@ pub fn decoder(dyn) {
 }
 
 pub fn variables(name: String, owner: String) {
- Some(
-  json.object([#("name", json.string(name)), #("owner", json.string(owner))]),
- )
+ let name = json.string(name)
+ let owner = json.string(owner)
+ Some(json.object([#("name", name), #("owner", owner)]))
 }

apps/backend/src/api/hex.gleam

@@ -9,31 +9,31 @@ import gleam/result
 import gleam/uri
 
 pub fn get_package_owners(package_name: String, secret hex_api_key: String) {
- use response <- result.try(
+ use response <- result.try({
  request.new()
  |> request.set_host("hex.pm")
  |> request.set_path("/api/packages/" <> package_name <> "/owners")
  |> request.prepend_header("authorization", hex_api_key)
- |> request.prepend_header("user-agent", "gloogle / 0.0.0")
+ |> request.prepend_header("user-agent", "gloogle / 1.0.0")
  |> httpc.send()
- |> result.map_error(error.FetchError),
- )
+ |> result.map_error(error.FetchError)
+ })
 
  response.body
  |> json.decode(using: dynamic.list(decode_hex_owner))
  |> result.map_error(error.JsonError)
 }
 
 pub fn get_package(package_name: String, secret hex_api_key: String) {
- use response <- result.try(
+ use response <- result.try({
  request.new()
  |> request.set_host("hex.pm")
  |> request.set_path("/api/packages/" <> package_name)
  |> request.prepend_header("authorization", hex_api_key)
- |> request.prepend_header("user-agent", "gloogle / 0.0.0")
+ |> request.prepend_header("user-agent", "gloogle / 1.0.0")
  |> httpc.send()
- |> result.map_error(error.FetchError),
- )
+ |> result.map_error(error.FetchError)
+ })
 
  response.body
  |> json.decode(using: hexpm.decode_package)
@@ -52,35 +52,33 @@ fn decode_hex_owner(data) {
 pub fn lookup_release(release: hexpm.PackageRelease, secret hex_api_key: String) {
  let assert Ok(url) = uri.parse(release.url)
 
- use response <- result.try(
+ use response <- result.try({
  request.new()
  |> request.set_host("hex.pm")
  |> request.set_path(url.path)
  |> request.prepend_header("authorization", hex_api_key)
- |> request.prepend_header("user-agent", "gloogle / 0.0.0")
+ |> request.prepend_header("user-agent", "gloogle / 1.0.0")
  |> httpc.send()
- |> result.map_error(error.FetchError),
- )
+ |> result.map_error(error.FetchError)
+ })
 
  response.body
  |> json.decode(using: hexpm.decode_release)
  |> result.map_error(error.JsonError)
 }
 
 pub fn get_api_packages_page(page: Int, hex_api_key: String) {
- use response <- result.try(
+ let page = int.to_string(page)
+ use response <- result.try({
  request.new()
  |> request.set_host("hex.pm")
  |> request.set_path("/api/packages")
- |> request.set_query([
- #("sort", "updated_at"),
- #("page", int.to_string(page)),
- ])
+ |> request.set_query([#("sort", "updated_at"), #("page", page)])
  |> request.prepend_header("authorization", hex_api_key)
- |> request.prepend_header("user-agent", "gloogle / 0.0.0")
+ |> request.prepend_header("user-agent", "gloogle / 1.0.0")
  |> httpc.send()
- |> result.map_error(error.FetchError),
- )
+ |> result.map_error(error.FetchError)
+ })
 
  response.body
  |> json.decode(using: dynamic.list(of: hexpm.decode_package))