Skip to content

Commit

Permalink
Improve collection handling in OAI-PMH
Browse files Browse the repository at this point in the history 
Remove the need for a full rescan
  • Loading branch information
@melmothx
melmothx committed Nov 21, 2023
1 parent db25b5a commit fe0f898
Show file tree
Hide file tree
Showing 5 changed files with 124 additions and 40 deletions.
5 changes: 2 additions & 3 deletions lib/AmuseWikiFarm/Archive/OAI/PMH.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,7 @@ sub update_site_records {
}
# create the sets for nodes here. Build a tree of them for fast lookup
my $node_tree = $site->node_title_tree->{titles};



# Dlog_debug { "Node tree is $_" } [ $node_tree, [ keys %node_sets ] ];

my @files;

Expand Down Expand Up @@ -198,6 +196,7 @@ sub update_site_records {
$f->{metadata_type} = $dc_type;
my $sets = delete $f->{sets} || [];
my $rec = $site->oai_pmh_records->update_or_create($f, { key => 'identifier_site_id_unique' });
Dlog_debug { "Setting the record's sets $_" } [ map { $_->set_spec } @$sets ] if @$sets;
$rec->set_oai_pmh_sets($sets);
}
}
Expand Down
6 changes: 3 additions & 3 deletions lib/AmuseWikiFarm/Controller/Nodes.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ sub create :Chained('admin') :PathPart('create') :Args(0) {
if (my $uri = $params{uri}) {
log_info { $c->user->get('username') . " is creating nodes/$uri" };
if (my $node = $site->nodes->update_or_create_from_params(\%params, { create => 1 })) {
$c->flash(status_msg => 'COLLECTION_UPDATE');
$c->flash(status_msg => $c->loc("Collection created"));
return $c->response->redirect($c->uri_for($node->full_uri));
}
else {
Expand Down Expand Up @@ -130,12 +130,12 @@ sub update_node :Chained('edit') :PathPart('') :Args(0) {
if ($params{update}) {
Dlog_info { $c->user->get('username') . " is updating " . $node->full_uri . " with $_" } \%params;
$node->update_from_params(\%params);
$c->flash(status_msg => 'COLLECTION_UPDATE');
$c->flash(status_msg => $c->loc("Collection has been updated"));
}
elsif ($params{delete}) {
Dlog_info { $c->user->get('username') . " deleted $_" } +{ $node->get_columns };
$node->delete;
$c->flash(status_msg => 'COLLECTION_UPDATE');
$c->flash(status_msg => $c->loc("Collection has been deleted"));
$c->response->redirect($c->uri_for_action('/nodes/node_root'));
return;
}
Expand Down
64 changes: 36 additions & 28 deletions lib/AmuseWikiFarm/Schema/Result/Node.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,12 +284,7 @@ sub is_root {
return !shift->parent_node_id;
}

has ancestors_cache => (is => 'ro',
lazy => 1,
isa => 'ArrayRef',
builder => '_build_ancestors_cache');

sub _build_ancestors_cache {
sub ancestors {
my $self = shift;
my @ancestors;
my $rec = $self;
Expand All @@ -298,11 +293,7 @@ sub _build_ancestors_cache {
while (++$max < 10 and $rec = $rec->parent) {
push @ancestors, $rec;
}
return \@ancestors;
}

sub ancestors {
return @{shift->ancestors_cache};
return @ancestors;
}

sub full_uri {
Expand All @@ -312,17 +303,21 @@ sub full_uri {

sub update_from_params {
my ($self, $params) = @_;
$params ||= {};
Dlog_debug { "Updating " . $self->full_uri . " with $_" } $params;
my $site = $self->site;
my @locales = $site->supported_locales;
my $guard = $self->result_source->schema->txn_scope_guard;
my $oai_pmh_set = $site->oai_pmh_sets->find_or_create({
set_spec => $self->oai_pmh_set_spec,
set_name => $self->canonical_title,
},
{ key => 'set_spec_site_id_unique' });
my %nodes;
foreach my $n ($self, $self->ancestors) {
$nodes{$n->node_id} = $site->oai_pmh_sets->find_or_create({
set_spec => $n->oai_pmh_set_spec,
set_name => $n->canonical_title,
},
{ key => 'set_spec_site_id_unique' });
}
# collect existing records. We will need to bump them.
my @oai_pmh_record_ids = map { $_->oai_pmh_record_id } $oai_pmh_set->oai_pmh_record_sets->all;
my @oai_pmh_record_ids = map { $_->oai_pmh_record_id } map { $_->oai_pmh_record_sets->all } values %nodes;

LANG:
foreach my $lang (@locales) {
Expand All @@ -339,11 +334,13 @@ sub update_from_params {
$body{body_html} = muse_to_object($body{body_muse})->as_html;
$self->node_bodies->update_or_create(\%body);
}
if (my $parent = $site->nodes->find_by_uri($params->{parent_node_uri})) {
$self->parent_node($parent);
}
else {
$self->parent_node(undef);
if (defined $params->{parent_node_uri}) {
if (my $parent = $site->nodes->find_by_uri($params->{parent_node_uri})) {
$self->parent_node($parent);
}
else {
$self->parent_node(undef);
}
}
if (defined $params->{sorting_pos} and $params->{sorting_pos} =~ m/\A[1-9][0-9]*\z/) {
log_debug { "Setting sorting pos to $params->{sorting_pos}" };
Expand Down Expand Up @@ -387,12 +384,12 @@ sub update_from_params {
# bumps the new ones.
my $tree = $site->node_title_tree;
my $self_node_id = $self->node_id;
my ($tree_spec) = grep { $_->{node_id} == $self_node_id } @{ $tree->{nodes} || []};
Dlog_debug { "Initial list of PMH records is $_" } \@oai_pmh_record_ids;
if ($tree_spec) {
foreach my $tree_spec (grep { $nodes{$_->{node_id}} } @{ $tree->{nodes} || []}) {
Dlog_debug { "Updating $_" } $tree_spec;
my @new_oai_pmh_records = $site->oai_pmh_records->by_title_id($tree_spec->{title_ids})->landing_pages_only->all;
# this should clear the existing one and relink
$oai_pmh_set->set_oai_pmh_records(\@new_oai_pmh_records);
$nodes{$tree_spec->{node_id}}->set_oai_pmh_records(\@new_oai_pmh_records);
push @oai_pmh_record_ids, map { $_->oai_pmh_record_id } @new_oai_pmh_records;
}
Dlog_debug { "Final list of PMH records is $_" } \@oai_pmh_record_ids;
Expand Down Expand Up @@ -592,13 +589,13 @@ sub title_ids {
columns => [qw/id/],
%hri,
});
Dlog_debug { "Direct ids for " . $self->uri . " are $_" } \@ids;
# Dlog_debug { "Direct ids for " . $self->uri . " are $_" } \@ids;
my @catids = map { $_->{title_id} } $self->categories->search_related('title_categories')->search(undef,
{
columns => [qw/title_id/],
%hri,
});
Dlog_debug { "Ids via category for " . $self->uri . " are $_" } \@catids;
# Dlog_debug { "Ids via category for " . $self->uri . " are $_" } \@catids;
return [ @ids, @catids];
}

Expand All @@ -618,7 +615,18 @@ sub oai_pmh_set_spec {
}

after insert => sub {
shift->update_full_path;
my $self = shift;
$self->update_full_path;
$self->update_from_params;
};

before delete => sub {
my $self = shift;
if (my $oaipmh_set = $self->site->oai_pmh_sets->find({ set_spec => $self->oai_pmh_set_spec })) {
$self->update_from_params;
log_debug { "Deleting the OAI_PMH set" };
$oaipmh_set->delete;
}
};

__PACKAGE__->meta->make_immutable;
Expand Down
4 changes: 0 additions & 4 deletions root/src/layout.tt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -698,10 +698,6 @@
<a href="[% c.uri_for_action('/bookbuilder/index') %]" class="text-success">
<strong>[% loc('The text was added to the bookbuilder') %]</strong>
</a>
[% ELSIF c.flash.status_msg == 'COLLECTION_UPDATE' %]
<a href="[% c.uri_for_action('/nodes/refresh_oai_pmh_repo') %]" class="text-success">
<strong>[% loc('Collections have been updated. You may want to refresh the OAI-PMH repository') %]</strong>
</a>
[% ELSE %]
[% c.flash.status_msg | html %]
[% END %]
Expand Down
85 changes: 83 additions & 2 deletions t/nodes.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use File::Spec::Functions qw/catfile catdir/;
use lib catdir(qw/t lib/);
use AmuseWikiFarm::Schema;
use Data::Dumper::Concise;
use Test::More tests => 35;
use Test::More tests => 64;
use AmuseWikiFarm::Archive::OAI::PMH;

my $builder = Test::More->builder;
Expand Down Expand Up @@ -131,7 +131,7 @@ foreach my $node ($site->nodes) {
}

# now we need the tree for each title
$site->node_title_tree;
diag Dumper($site->node_title_tree);

my $oai_pmh = AmuseWikiFarm::Archive::OAI::PMH->new(site => $site,
oai_pmh_url => URI->new($site->canonical_url . '/oai-pmh'));
Expand Down Expand Up @@ -196,4 +196,85 @@ foreach my $set ("category:author:author-one-1",
diag $mech->content;
$mech->content_lacks('noRecordsMatch');

sleep 1;
$now = DateTime->now(time_zone => 'UTC');

$uri->query_form({ from => $now->iso8601 . 'Z',
metadataPrefix => 'oai_dc', verb => 'ListIdentifiers' });
$mech->get_ok($uri);
$mech->content_contains('noRecordsMatch');

# create one
my $root_node = $site->nodes->create({ uri => 'root-oai-pmh' });
my $child_node = $site->nodes->create({
uri => 'child-oai-pmh',
parent_node => $root_node,
});
$root_node->update_from_params({
attached_uris => "/library/four-2",
});

$child_node->update_from_params({
attached_uris => "/library/seven /category/topic/cat-one-1",
});
sleep 1;
$mech->get_ok($uri);
$mech->content_lacks('noRecordsMatch');
$mech->content_contains('oai:0nodes1.amusewiki.org:/library/one-1', "one-1 is there because of category");
$mech->content_contains('oai:0nodes1.amusewiki.org:/library/seven', "seven is there because of direct child");

# now, ask for the root
$uri->query_form({ from => $now->iso8601 . 'Z',
metadataPrefix => 'oai_dc', verb => 'ListIdentifiers',
set => 'collection:root-oai-pmh',
});
$mech->get_ok($uri);
$mech->content_contains('oai:0nodes1.amusewiki.org:/library/one-1', "one-1 is there because of category");
$mech->content_contains('oai:0nodes1.amusewiki.org:/library/seven', "seven is there because of direct child");
diag $mech->content;
# diag Dumper($site->node_title_tree);
sleep 1;
$now = DateTime->now(time_zone => 'UTC');
$uri->query_form({ from => $now->iso8601 . 'Z',
metadataPrefix => 'oai_dc', verb => 'ListIdentifiers' });

$mech->get_ok($uri);
$mech->content_contains('noRecordsMatch');

# now update the child and ask for the parent
$child_node->update_from_params({ attached_uris => "/library/one-2" });

$uri->query_form({ from => $now->iso8601 . 'Z',
metadataPrefix => 'oai_dc', verb => 'ListIdentifiers' });
$mech->get_ok($uri);
$mech->content_contains('oai:0nodes1.amusewiki.org:/library/one-1', "one-1 is there because of category");
$mech->content_contains('oai:0nodes1.amusewiki.org:/library/seven', "seven is there because of direct child");
$mech->content_contains('oai:0nodes1.amusewiki.org:/library/one-2', "one-2 is there because of category");
$mech->content_contains('oai:0nodes1.amusewiki.org:/library/four-2', "four-2 is attached to the parent");


foreach my $set ('collection:root-oai-pmh', 'collection:child-oai-pmh') {
$uri->query_form({ from => $now->iso8601 . 'Z',
metadataPrefix => 'oai_dc', verb => 'ListIdentifiers',
set => $set,
});
$mech->get_ok($uri);
$mech->content_lacks('oai:0nodes1.amusewiki.org:/library/one-1', "one-1 is there because of category");
$mech->content_lacks('oai:0nodes1.amusewiki.org:/library/seven', "seven is there because of direct child");
$mech->content_contains('oai:0nodes1.amusewiki.org:/library/one-2', "one-2 is there because of category");
diag $mech->content;
}
sleep 1;
diag "Trying a deletion now";
$now = DateTime->now(time_zone => 'UTC');
$uri->query_form({ from => $now->iso8601 . 'Z',
metadataPrefix => 'oai_dc', verb => 'ListIdentifiers' });
$mech->get_ok($uri);
$mech->content_contains('noRecordsMatch');

$child_node->delete;
$mech->get_ok($uri);
$mech->content_contains('oai:0nodes1.amusewiki.org:/library/one-2', "one-2 is there because of category");
$mech->content_contains('oai:0nodes1.amusewiki.org:/library/four-2', "four-2 is attached to the parent");
diag $mech->content;
}

0 comments on commit fe0f898

Please sign in to comment.