Small fix for web archive results beyond first page + tests

src/data-source/collection-browser-data-source-interface.ts

@@ -219,6 +219,11 @@ export interface CollectionBrowserDataSourceInterface
  */
  refreshLetterCounts(): void;
 
+ /**
+ * Returns the current page size of the data source.
+ */
+ getPageSize(): number;
+
  /**
  * Changes the page size used by the data source, discarding any previously-fetched pages.
  *

src/data-source/collection-browser-data-source.ts

@@ -22,7 +22,7 @@ import {
  SortField,
  SORT_OPTIONS,
 } from '../models';
-import type { PageSpecifierParams } from './models';
+import { FACETLESS_PAGE_ELEMENTS, type PageSpecifierParams } from './models';
 import type { CollectionBrowserDataSourceInterface } from './collection-browser-data-source-interface';
 import type { CollectionBrowserSearchInterface } from './collection-browser-query-state';
 import { sha1 } from '../utils/sha1';
@@ -31,10 +31,21 @@ import { log } from '../utils/log';
 export class CollectionBrowserDataSource
  implements CollectionBrowserDataSourceInterface
 {
+ /**
+ * All pages of tile models that have been fetched so far, indexed by their page
+ * number (with the first being page 1).
+ */
  private pages: Record<string, TileModel[]> = {};
 
+ /**
+ * Tile offset to apply when looking up tiles in the pages, in order to maintain
+ * page alignment after tiles are removed.
+ */
  private offset = 0;
 
+ /**
+ * Total number of tile models stored in this data source's pages
+ */
  private numTileModels = 0;
 
  /**
@@ -48,10 +59,21 @@ export class CollectionBrowserDataSource
  */
  private previousQueryKey: string = '';
 
+ /**
+ * Whether the initial page of search results for the current query state
+ * is presently being fetched.
+ */
  private searchResultsLoading = false;
 
+ /**
+ * Whether the facets (aggregations) for the current query state are
+ * presently being fetched.
+ */
  private facetsLoading = false;
 
+ /**
+ * Whether further query changes should be ignored and not trigger fetches
+ */
  private suppressFetches = false;
 
  /**
@@ -265,6 +287,13 @@ export class CollectionBrowserDataSource
  return Object.values(this.pages).flat().indexOf(tile);
  }
 
+ /**
+ * @inheritdoc
+ */
+ getPageSize(): number {
+ return this.pageSize;
+ }
+
  /**
  * @inheritdoc
  */
@@ -294,11 +323,15 @@ export class CollectionBrowserDataSource
  this._initialSearchCompleteResolver = res;
  });
 
+ const shouldFetchFacets =
+ !this.host.suppressFacets &&
+ !FACETLESS_PAGE_ELEMENTS.includes(this.host.profileElement!);
+
  // Fire the initial page & facet requests
  this.queryInitialized = true;
  await Promise.all([
  this.doInitialPageFetch(),
- this.host.suppressFacets ? null : this.fetchFacets(),
+ shouldFetchFacets ? this.fetchFacets() : null,
  ]);
 
  // Resolve the `initialSearchComplete` promise for this search
@@ -876,7 +909,7 @@ export class CollectionBrowserDataSource
 
  // Batch multiple initial page requests together if needed (e.g., can request
  // pages 1 and 2 together in a single request).
- const numPages = pageNumber === 1 ? numInitialPages : 1;
+ let numPages = pageNumber === 1 ? numInitialPages : 1;
  const numRows = this.pageSize * numPages;
 
  // if a fetch is already in progress for this query and page, don't fetch again
@@ -990,7 +1023,16 @@ export class CollectionBrowserDataSource
  }
  }
 
- // Update the data source for each returned page
+ // Update the data source for each returned page.
+ // For loans and web archives, we must account for receiving more pages than we asked for.
+ if (
+ this.host.profileElement === 'lending' ||
+ this.host.profileElement === 'web_archives'
+ ) {
+ numPages = Math.ceil(results.length / this.pageSize);
+ this.endOfDataReached = true;
+ if (this.activeOnHost) this.host.setTileCount(this.totalResults);
+ }
  for (let i = 0; i < numPages; i += 1) {
  const pageStartIndex = this.pageSize * i;
  this.addTilesToDataSource(

src/data-source/models.ts

@@ -27,3 +27,12 @@ export type PageSpecifierParams = {
  */
  pageElements?: PageElementName[];
 };
+
+/**
+ * List of profile page elements that do not currently allow faceting
+ */
+export const FACETLESS_PAGE_ELEMENTS: PageElementName[] = [
+ 'forum_posts',
+ 'lending',
+ 'web_archives',
+];

test/collection-browser.test.ts

@@ -1562,6 +1562,37 @@ describe('Collection Browser', () => {
  expect(el.maxSelectedDate).not.to.exist;
  });
 
+ it('correctly retrieves web archive hits', async () => {
+ const searchService = new MockSearchService();
+ const el = await fixture<CollectionBrowser>(
+ html`<collection-browser
+ .searchService=${searchService}
+ .withinProfile=${'@foo'}
+ .profileElement=${'web_archives'}
+ >
+ </collection-browser>`
+ );
+
+ el.baseQuery = 'web-archive';
+ await el.updateComplete;
+ await el.initialSearchComplete;
+ await nextTick();
+
+ console.log(
+ '\n\n*****\n\n*****\n\n',
+ el.dataSource.getAllPages(),
+ '\n\n*****\n\n*****\n\n'
+ );
+ expect(el.dataSource.totalResults, 'total results').to.equal(1);
+ expect(el.dataSource.getTileModelAt(0)?.title).to.equal(
+ 'https://example.com'
+ );
+ expect(
+ el.dataSource.getTileModelAt(0)?.captureDates?.length,
+ 'capture dates'
+ ).to.equal(1);
+ });
+
  it('shows temporarily unavailable message when facets suppressed', async () => {
  const searchService = new MockSearchService();
  const el = await fixture<CollectionBrowser>(

test/data-source/collection-browser-data-source.test.ts

@@ -1,10 +1,12 @@
 import { expect, fixture } from '@open-wc/testing';
 import { html } from 'lit';
+import sinon from 'sinon';
 import { ItemHit, SearchType } from '@internetarchive/search-service';
 import { CollectionBrowserDataSource } from '../../src/data-source/collection-browser-data-source';
 import { TileModel } from '../../src/models';
 import type { CollectionBrowser } from '../../src/collection-browser';
 import '../../src/collection-browser';
+import { MockSearchService } from '../mocks/mock-search-service';
 
 const dataPage: TileModel[] = [
  new TileModel(
@@ -32,7 +34,7 @@ describe('Collection Browser Data Source', () => {
  `);
  });
 
- it('can add and retrieve data pages', async () => {
+ it('can add and retrieve data pages', () => {
  const dataSource = new CollectionBrowserDataSource(host);
  dataSource.addPage(1, dataPage);
 
@@ -42,12 +44,13 @@ describe('Collection Browser Data Source', () => {
  expect(dataSource.getPage(1)[1].identifier).to.equal('bar');
  });
 
- it('resets data when changing page size', async () => {
+ it('resets data when changing page size', () => {
  const dataSource = new CollectionBrowserDataSource(host);
  dataSource.addPage(1, dataPage);
 
  dataSource.setPageSize(100);
  expect(Object.keys(dataSource.getAllPages()).length).to.equal(0);
+ expect(dataSource.getPageSize()).to.equal(100);
  });
 
  it('can be installed on the host', async () => {
@@ -70,7 +73,21 @@ describe('Collection Browser Data Source', () => {
  host.removeController(dataSource);
  });
 
- it('refreshes prefix filter counts', async () => {
+ it('can suppress further fetches', async () => {
+ host.searchService = new MockSearchService();
+
+ const pageFetchSpy = sinon.spy();
+ const dataSource = new CollectionBrowserDataSource(host);
+ dataSource.fetchPage = pageFetchSpy;
+
+ dataSource.addPage(1, dataPage);
+ dataSource.setFetchesSuppressed(true);
+ dataSource.handleQueryChange();
+
+ expect(pageFetchSpy.callCount).to.equal(0);
+ });
+
+ it('refreshes prefix filter counts', () => {
  const dataSource = new CollectionBrowserDataSource(host);
  dataSource.addPage(1, dataPage);
 

test/mocks/mock-search-responses.ts

@@ -6,6 +6,7 @@ import {
  SearchServiceError,
  TextHit,
 } from '@internetarchive/search-service';
+import { WebArchiveHit } from '@internetarchive/search-service/dist/src/models/hit-types/web-archive-hit';
 import { SearchServiceErrorType } from '@internetarchive/search-service/dist/src/search-service-error';
 
 export const getMockSuccessSingleResult: () => Result<
@@ -876,6 +877,49 @@ export const getMockSuccessWithParentCollections: () => Result<
  },
 });
 
+export const getMockSuccessWithWebArchiveHits: () => Result<
+ SearchResponse,
+ SearchServiceError
+> = () => ({
+ success: {
+ request: {
+ kind: 'hits',
+ clientParameters: {
+ user_query: 'web-archive',
+ sort: [],
+ },
+ backendRequests: {
+ primary: {
+ kind: 'hits',
+ finalized_parameters: {
+ user_query: 'web-archive',
+ sort: [],
+ },
+ },
+ },
+ },
+ rawResponse: {},
+ response: {
+ totalResults: 1,
+ returnedCount: 1,
+ results: [
+ new WebArchiveHit({
+ fields: {
+ url: 'https://example.com',
+ capture_dates: ['2010-01-02T03:04:05Z'],
+ __href__:
+ 'https://web.archive.org/web/20100102030405/https%3A%2F%2Fexample.com',
+ },
+ }),
+ ],
+ },
+ responseHeader: {
+ succeeded: true,
+ query_time: 0,
+ },
+ },
+});
+
 export const getMockErrorResult: () => Result<
  SearchResponse,
  SearchServiceError

test/mocks/mock-search-service.ts

@@ -28,6 +28,7 @@ import {
  getMockSuccessWithParentCollections,
  getMockSuccessManyFields,
  getMockSuccessNoResults,
+ getMockSuccessWithWebArchiveHits,
 } from './mock-search-responses';
 
 const responses: Record<
@@ -49,6 +50,7 @@ const responses: Record<
  'default-sort-concise': getMockSuccessWithConciseDefaultSort,
  'fav-sort': getMockSuccessWithDefaultFavSort,
  'parent-collections': getMockSuccessWithParentCollections,
+ 'web-archive': getMockSuccessWithWebArchiveHits,
  'many-fields': getMockSuccessManyFields,
  'no-results': getMockSuccessNoResults,
  error: getMockErrorResult,