Skip to content

Commit

Permalink
webapp: Implement repository graphs (for absolute values only)
Browse files Browse the repository at this point in the history 
Also implement a way to use new history for these (#120)
  • Loading branch information
@AMDmi3
AMDmi3 committed Nov 19, 2024
1 parent 45b6c71 commit 6696695
Show file tree
Hide file tree
Showing 5 changed files with 402 additions and 27 deletions.
35 changes: 19 additions & 16 deletions repology-webapp/src/endpoints.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,23 @@ pub enum Endpoint {
#[strum(props(path = "/graph/total/problems.svg"))]
GraphTotalProblems,

#[strum(props(path = "/graph/repo/:repository_name/problems.svg"))]
GraphRepoProblems,
#[strum(props(path = "/graph/repo/:repository_name/maintainers.svg"))]
GraphRepoMaintainers,
#[strum(props(path = "/graph/repo/:repository_name/projects_total.svg"))]
GraphRepoProjectsTotal,
#[strum(props(path = "/graph/repo/:repository_name/projects_unique.svg"))]
GraphRepoProjectsUnique,
#[strum(props(path = "/graph/repo/:repository_name/projects_newest.svg"))]
GraphRepoProjectsNewest,
#[strum(props(path = "/graph/repo/:repository_name/projects_outdated.svg"))]
GraphRepoProjectsOutdated,
#[strum(props(path = "/graph/repo/:repository_name/projects_problematic.svg"))]
GraphRepoProjectsProblematic,
#[strum(props(path = "/graph/repo/:repository_name/projects_vulnerable.svg"))]
GraphRepoProjectsVulnerable,

// Opensearch
#[strum(props(path = "/opensearch/project.xml"))]
OpensearchProject,
Expand Down Expand Up @@ -238,40 +255,26 @@ pub enum Endpoint {
// Graphs
#[strum(props(path = "/graph/project/:project_name/releases.svg"))]
GraphReleases,

#[strum(props(path = "/graph/map_repo_size_fresh.svg"))]
GraphMapRepoSizeFresh,
#[strum(props(path = "/graph/map_repo_size_fresh_nonunique.svg"))]
GraphMapRepoSizeFreshNonunique,
#[strum(props(path = "/graph/map_repo_size_freshness.svg"))]
GraphMapRepoSizeFreshness,
#[strum(props(path = "/graph/repo/:repository_name/projects_total.svg"))]
GraphRepoProjectsTotal,
#[strum(props(path = "/graph/repo/:repository_name/projects_newest.svg"))]
GraphRepoProjectsNewest,

#[strum(props(path = "/graph/repo/:repository_name/projects_newest_percent.svg"))]
GraphRepoProjectsNewestPercent,
#[strum(props(path = "/graph/repo/:repository_name/projects_outdated.svg"))]
GraphRepoProjectsOutdated,
#[strum(props(path = "/graph/repo/:repository_name/projects_outdated_percent.svg"))]
GraphRepoProjectsOutdatedPercent,
#[strum(props(path = "/graph/repo/:repository_name/projects_unique.svg"))]
GraphRepoProjectsUnique,
#[strum(props(path = "/graph/repo/:repository_name/projects_unique_percent.svg"))]
GraphRepoProjectsUniquePercent,
#[strum(props(path = "/graph/repo/:repository_name/projects_problematic.svg"))]
GraphRepoProjectsProblematic,
#[strum(props(path = "/graph/repo/:repository_name/projects_problematic_percent.svg"))]
GraphRepoProjectsProblematicPercent,
#[strum(props(path = "/graph/repo/:repository_name/projects_vulnerable.svg"))]
GraphRepoProjectsVulnerable,
#[strum(props(path = "/graph/repo/:repository_name/projects_vulnerable_percent.svg"))]
GraphRepoProjectsVulnerablePercent,
#[strum(props(path = "/graph/repo/:repository_name/problems.svg"))]
GraphRepoProblems,
#[strum(props(path = "/graph/repo/:repository_name/problems_per_metapackage.svg"))]
GraphRepoProblemsPerMetapackage,
#[strum(props(path = "/graph/repo/:repository_name/maintainers.svg"))]
GraphRepoMaintainers,
#[strum(props(path = "/graph/repo/:repository_name/packages_per_maintainer.svg"))]
GraphRepoPackagesPerMaintainer,

Expand Down
8 changes: 8 additions & 0 deletions repology-webapp/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,14 @@ pub async fn create_app(pool: PgPool) -> Result<Router, Error> {
.route(GraphTotalProjects.path(), get(views::graph_total_projects))
.route(GraphTotalMaintainers.path(), get(views::graph_total_maintainers))
.route(GraphTotalProblems.path(), get(views::graph_total_problems))
.route(GraphRepoProblems.path(), get(views::graph_repository_problems))
.route(GraphRepoMaintainers.path(), get(views::graph_repository_maintainers))
.route(GraphRepoProjectsTotal.path(), get(views::graph_repository_projects_total))
.route(GraphRepoProjectsUnique.path(), get(views::graph_repository_projects_unique))
.route(GraphRepoProjectsNewest.path(), get(views::graph_repository_projects_newest))
.route(GraphRepoProjectsOutdated.path(), get(views::graph_repository_projects_outdated))
.route(GraphRepoProjectsProblematic.path(), get(views::graph_repository_projects_problematic))
.route(GraphRepoProjectsVulnerable.path(), get(views::graph_repository_projects_vulnerable))
.route(Log.path(), get(views::log))
.route(MaintainerRepoFeed.path(), get(views::maintainer_repo_feed))
.route(MaintainerRepoFeedAtom.path(), get(views::maintainer_repo_feed_atom))
Expand Down
233 changes: 229 additions & 4 deletions repology-webapp/src/views/graph.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@

use std::time::Duration;

use axum::extract::State;
use axum::http::{header, HeaderValue};
use axum::extract::{Path, Query, State};
use axum::http::{header, HeaderValue, StatusCode};
use axum::response::IntoResponse;
use chrono::{DateTime, Utc};
use indoc::indoc;
use serde::Deserialize;
use sqlx::PgPool;

use crate::graphs::{render_graph, GraphType};
Expand All @@ -16,7 +17,231 @@ use crate::state::AppState;

const GRAPH_PERIOD: Duration = Duration::from_days(21);

async fn graph_total_generic(pool: &PgPool, field: &str, stroke: &str) -> EndpointResult {
#[derive(Deserialize, Debug)]
pub struct QueryParams {
#[serde(default)]
#[serde(deserialize_with = "crate::query::deserialize_bool_flag")]
pub experimental_history: bool,
}

async fn graph_repository_generic(
state: &AppState,
repository_name: &str,
experimental_history: bool,
field_name: &str,
stroke: &str,
) -> EndpointResult {
if state
.repository_data_cache
.get_active(&repository_name)
.await
.is_none()
{
return Ok((StatusCode::NOT_FOUND, "repository not found".to_owned()).into_response());
};

let query = if experimental_history {
&format!(
indoc! {r#"
(
SELECT
ts AS timestamp,
{0}::real AS value
FROM repositories_history_new
WHERE repository_id = (SELECT id FROM repositories WHERE name = $1) AND ts < now() - $3 AND {0} IS NOT NULL
ORDER BY ts DESC
LIMIT 1
)
UNION ALL
(
SELECT
ts AS timestamp,
{0}::real AS value
FROM repositories_history_new
WHERE repository_id = (SELECT id FROM repositories WHERE name = $1) AND ts >= now() - $3 AND {0} IS NOT NULL
ORDER BY ts
)
"#},
field_name
)
} else {
indoc! {r#"
(
SELECT
ts AS timestamp,
(snapshot->$1->>$2)::real AS value
FROM repositories_history
WHERE ts < now() - $3 AND snapshot->$1->>$2 IS NOT NULL
ORDER BY ts DESC
LIMIT 1
)
UNION ALL
(
SELECT
ts AS timestamp,
(snapshot->$1->>$2)::real AS value
FROM repositories_history
WHERE ts >= now() - $3 AND snapshot->$1->>$2 IS NOT NULL
ORDER BY ts
)
"#}
};
let points: Vec<(DateTime<Utc>, f32)> = sqlx::query_as(query)
.bind(&repository_name)
.bind(&field_name.replace("num_projects", "num_metapackages"))
.bind(&GRAPH_PERIOD)
.fetch_all(&state.pool)
.await?;

let now = Utc::now();

Ok((
[(
header::CONTENT_TYPE,
HeaderValue::from_static(mime::IMAGE_SVG.as_ref()),
)],
render_graph(
&points
.into_iter()
.map(|(timestamp, value)| ((now - timestamp).to_std().unwrap(), value))
.collect(),
GraphType::Integer,
GRAPH_PERIOD,
stroke,
),
)
.into_response())
}

#[cfg_attr(not(feature = "coverage"), tracing::instrument(skip(state)))]
pub async fn graph_repository_maintainers(
Path(repository_name): Path<String>,
Query(query): Query<QueryParams>,
State(state): State<AppState>,
) -> EndpointResult {
graph_repository_generic(
&state,
&repository_name,
query.experimental_history,
"num_maintainers",
"#c000c0",
)
.await
}

#[cfg_attr(not(feature = "coverage"), tracing::instrument(skip(state)))]
pub async fn graph_repository_problems(
Path(repository_name): Path<String>,
Query(query): Query<QueryParams>,
State(state): State<AppState>,
) -> EndpointResult {
graph_repository_generic(
&state,
&repository_name,
query.experimental_history,
"num_problems",
"#c00000",
)
.await
}

#[cfg_attr(not(feature = "coverage"), tracing::instrument(skip(state)))]
pub async fn graph_repository_projects_total(
Path(repository_name): Path<String>,
Query(query): Query<QueryParams>,
State(state): State<AppState>,
) -> EndpointResult {
graph_repository_generic(
&state,
&repository_name,
query.experimental_history,
"num_projects",
"#000",
)
.await
}

#[cfg_attr(not(feature = "coverage"), tracing::instrument(skip(state)))]
pub async fn graph_repository_projects_unique(
Path(repository_name): Path<String>,
Query(query): Query<QueryParams>,
State(state): State<AppState>,
) -> EndpointResult {
graph_repository_generic(
&state,
&repository_name,
query.experimental_history,
"num_projects_unique",
"#5bc0de",
)
.await
}

#[cfg_attr(not(feature = "coverage"), tracing::instrument(skip(state)))]
pub async fn graph_repository_projects_newest(
Path(repository_name): Path<String>,
Query(query): Query<QueryParams>,
State(state): State<AppState>,
) -> EndpointResult {
graph_repository_generic(
&state,
&repository_name,
query.experimental_history,
"num_projects_newest",
"#5cb85c",
)
.await
}

#[cfg_attr(not(feature = "coverage"), tracing::instrument(skip(state)))]
pub async fn graph_repository_projects_outdated(
Path(repository_name): Path<String>,
Query(query): Query<QueryParams>,
State(state): State<AppState>,
) -> EndpointResult {
graph_repository_generic(
&state,
&repository_name,
query.experimental_history,
"num_projects_outdated",
"#d9534f",
)
.await
}

#[cfg_attr(not(feature = "coverage"), tracing::instrument(skip(state)))]
pub async fn graph_repository_projects_problematic(
Path(repository_name): Path<String>,
Query(query): Query<QueryParams>,
State(state): State<AppState>,
) -> EndpointResult {
graph_repository_generic(
&state,
&repository_name,
query.experimental_history,
"num_projects_problematic",
"#808080",
)
.await
}

#[cfg_attr(not(feature = "coverage"), tracing::instrument(skip(state)))]
pub async fn graph_repository_projects_vulnerable(
Path(repository_name): Path<String>,
Query(query): Query<QueryParams>,
State(state): State<AppState>,
) -> EndpointResult {
graph_repository_generic(
&state,
&repository_name,
query.experimental_history,
"num_projects_vulnerable",
"#ff0000",
)
.await
}

async fn graph_total_generic(pool: &PgPool, field_name: &str, stroke: &str) -> EndpointResult {
let points: Vec<(DateTime<Utc>, f32)> = sqlx::query_as(indoc! {r#"
(
SELECT
Expand All @@ -37,7 +262,7 @@ async fn graph_total_generic(pool: &PgPool, field: &str, stroke: &str) -> Endpoi
ORDER BY ts
)
"#})
.bind(&field)
.bind(&field_name)
.bind(&GRAPH_PERIOD)
.fetch_all(pool)
.await?;
Expand Down
14 changes: 11 additions & 3 deletions repology-webapp/tests/fixtures/graphs_data.sql
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,12 @@
-- first point is intentionally long in the past to check whether it's handled properly producing a line
INSERT INTO statistics_history(ts, snapshot) VALUES
-- point is intentionally long in the past to check whether it's handled properly producing a line
(now() - interval '128 day', '{"num_packages": 0, "num_problems": 0, "num_maintainers": 0, "num_metapackages": 0}'),
(now(), '{"num_packages": 10, "num_problems": 10, "num_maintainers": 10, "num_metapackages": 10}');
(now() - interval '128 day', '{"num_packages":0,"num_problems":0,"num_maintainers":0,"num_metapackages":0}'),
(now(), '{"num_packages":10,"num_problems":10,"num_maintainers":10,"num_metapackages":10}');

INSERT INTO repositories_history(ts, snapshot) VALUES
(now() - interval '128 day', '{"freebsd":{"num_maintainers":0,"num_problems":0,"num_metapackages":0,"num_metapackages_unique":0,"num_metapackages_newest":0,"num_metapackages_outdated":0,"num_metapackages_problematic":0,"num_metapackages_vulnerable":0}}'::jsonb),
(now(), '{"freebsd":{"num_maintainers":10,"num_problems":10,"num_metapackages":10,"num_metapackages_unique":10,"num_metapackages_newest":10,"num_metapackages_outdated":10,"num_metapackages_problematic":10,"num_metapackages_vulnerable":10}}'::jsonb);

INSERT INTO repositories_history_new(repository_id, ts, num_maintainers, num_problems, num_projects, num_projects_unique, num_projects_newest, num_projects_outdated, num_projects_problematic, num_projects_vulnerable) VALUES
(1, now() - interval '128 day', 0, 0, 0, 0, 0, 0, 0, 0),
(1, now(), 0, 0, 0, 0, 0, 0, 0, 0);
Loading

0 comments on commit 6696695

Please sign in to comment.