Skip to content

Commit

Permalink
Add paging support for french doctolib + add tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@danburg @rbignon
danburg authored and rbignon committed Jul 8, 2021
1 parent ded2df3 commit 7bd2004
Show file tree
Hide file tree
Showing 2 changed files with 312 additions and 1 deletion.
19 changes: 19 additions & 0 deletions doctoshotgun.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,25 @@ def iter_centers_ids(self):
yield data['searchResultId']

def get_next_page(self):
# French doctolib uses data-u attribute of span-element to create the link when user hovers span
for span in self.doc.xpath('//div[contains(@class, "next")]/span'):
if not span.attrib.has_key('data-u'):
continue

# How to find the corresponding javascript-code:
# Press F12 to open dev-tools, select elements-tab, find div.next, right click on element and enable break on substructure change
# Hover "Next" element and follow callstack upwards
# JavaScript:
# var t = (e = r()(e)).data("u")
# , n = atob(t.replace(/\s/g, '').split('').reverse().join(''));

import base64
href = base64.urlsafe_b64decode(''.join(span.attrib['data-u'].split())[::-1]).decode()
query = dict(parse.parse_qsl(parse.urlsplit(href).query))

if 'page' in query:
return int(query['page'])

for a in self.doc.xpath('//div[contains(@class, "next")]/a'):
href = a.attrib['href']
query = dict(parse.parse_qsl(parse.urlsplit(href).query))
Expand Down
294 changes: 293 additions & 1 deletion test_browser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@
from requests.adapters import Response
import responses
from html import escape
import lxml.html as html
import json
import datetime
from woob.browser.browsers import Browser
from woob.browser.exceptions import ServerError
from doctoshotgun import DoctolibDE, DoctolibFR, CenterBookingPage
from doctoshotgun import CentersPage, DoctolibDE, DoctolibFR, CenterBookingPage

# globals
FIXTURES_FOLDER = "test_fixtures"
Expand Down Expand Up @@ -114,6 +115,297 @@ def test_find_centers_de_returns_502_should_fail(tmp_path):
pass


@responses.activate
def test_get_next_page_fr_should_return_2_on_page_1(tmp_path):
"""
Check that get_next_page returns 2 when we are on page 1 and there is a next page available
"""

"""
Next (data-u decoded): /vaccination-covid-19-autres-professions-prioritaires/france?page=2&ref_visit_motive_ids%5B%5D=6970&ref_visit_motive_ids%5B%5D=7005
"""

htmlString = """
<div class="next-previous-links">
<div class="previous dl-rounded-borders dl-white-bg">
<span class="disabled">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
Précédent
</span>
</div>
<div class="next dl-rounded-borders dl-white-bg">
<span data-u="=UDMwcTPEVTJCVTJzRWafVmdpR3bt9FdpNXa29lZlJnJwcTO20DR1UiQ
1Uyckl2XlZXa09WbfRXazlmdfZWZyZiM9U2ZhB3PlNmbhJnZvMXZylWY0lmc
vlmcw1ycu9WazNXZm9mcw1yclJHd1FWL5ETLklmdvNWLu9Wa0FmbpN2YhZ3L">
Suivant
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
</span>
</div>
</div>
"""
doc = html.document_fromstring(htmlString)

response = Response()
response._content = b'{}'

centers_page = CentersPage(browser=Browser(), response=response)
centers_page.doc = doc
next_page = centers_page.get_next_page()
assert next_page == 2


@responses.activate
def test_get_next_page_fr_should_return_3_on_page_2(tmp_path):
"""
Check that get_next_page returns 3 when we are on page 2 and next page is available
"""

"""
Previous (data-u decoded): /vaccination-covid-19-autres-professions-prioritaires/france?ref_visit_motive_ids%5B%5D=6970&ref_visit_motive_ids%5B%5D=7005
Next (data-u decoded): /vaccination-covid-19-autres-professions-prioritaires/france?page=3&ref_visit_motive_ids%5B%5D=6970&ref_visit_motive_ids%5B%5D=7005
"""

htmlString = """
<div class="next-previous-links">
<div class="previous dl-rounded-borders dl-white-bg">
<span data-u="==QNwAzN9QUNlIUNlMHZp9VZ2lGdv12X0l2cpZ3XmVmcmAzN
5YTPEVTJCVTJzRWafVmdpR3bt9FdpNXa29lZlJ3PlNmbhJnZvMXZylWY0lmc
vlmcw1ycu9WazNXZm9mcw1yclJHd1FWL5ETLklmdvNWLu9Wa0FmbpN2YhZ3L">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
Précédent
</span>
</div>
<div class="next dl-rounded-borders dl-white-bg">
<span data-u="=UDMwcTPEVTJCVTJzRWafVmdpR3bt9FdpNXa29lZlJnJwcTO20DR1UiQ
1Uyckl2XlZXa09WbfRXazlmdfZWZyZyM9U2ZhB3PlNmbhJnZvMXZylWY0lmc
vlmcw1ycu9WazNXZm9mcw1yclJHd1FWL5ETLklmdvNWLu9Wa0FmbpN2YhZ3L">
Suivant
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
</span>
</div>
</div>
"""
doc = html.document_fromstring(htmlString)

response = Response()
response._content = b'{}'

centers_page = CentersPage(browser=Browser(), response=response)
centers_page.doc = doc
next_page = centers_page.get_next_page()
assert next_page == 3


@responses.activate
def test_get_next_page_fr_should_return_4_on_page_3(tmp_path):
"""
Check that get_next_page returns 4 when we are on page 3 and next page is available
"""

"""
Previous (data-u decoded): /vaccination-covid-19-autres-professions-prioritaires/france?page=2&ref_visit_motive_ids%5B%5D=6970&ref_visit_motive_ids%5B%5D=7005
Next (data-u decoded): /vaccination-covid-19-autres-professions-prioritaires/france?page=4&ref_visit_motive_ids%5B%5D=6970&ref_visit_motive_ids%5B%5D=7005
"""

htmlString = """
<div class="next-previous-links">
<div class="previous dl-rounded-borders dl-white-bg">
<span data-u="=UDMwcTPEVTJCVTJzRWafVmdpR3bt9FdpNXa29lZlJnJwcTO20DR1UiQ
1Uyckl2XlZXa09WbfRXazlmdfZWZyZiM9U2ZhB3PlNmbhJnZvMXZylWY0lmc
vlmcw1ycu9WazNXZm9mcw1yclJHd1FWL5ETLklmdvNWLu9Wa0FmbpN2YhZ3L">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
Précédent
</span>
</div>
<div class="next dl-rounded-borders dl-white-bg">
<span data-u="=UDMwcTPEVTJCVTJzRWafVmdpR3bt9FdpNXa29lZlJnJwcTO20DR1UiQ
1Uyckl2XlZXa09WbfRXazlmdfZWZyZCN9U2ZhB3PlNmbhJnZvMXZylWY0lmc
vlmcw1ycu9WazNXZm9mcw1yclJHd1FWL5ETLklmdvNWLu9Wa0FmbpN2YhZ3L">
Suivant
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
</span>
</div>
</div>
"""
doc = html.document_fromstring(htmlString)

response = Response()
response._content = b'{}'

centers_page = CentersPage(browser=Browser(), response=response)
centers_page.doc = doc
next_page = centers_page.get_next_page()
assert next_page == 4


def test_get_next_page_fr_should_return_None_on_last_page(tmp_path):
"""
Check that get_next_page returns None when we are on the last page
"""
"""
Previous (data-u decoded): /vaccination-covid-19-autres-professions-prioritaires/france?page=7&ref_visit_motive_ids%5B%5D=6970&ref_visit_motive_ids%5B%5D=7005
"""

htmlString = """
<div class="next-previous-links">
<div class="previous dl-rounded-borders dl-white-bg">
<span data-u="=UDMwcTPEVTJCVTJzRWafVmdpR3bt9FdpNXa29lZlJnJwcTO20DR1UiQ
1Uyckl2XlZXa09WbfRXazlmdfZWZyZyN9U2ZhB3PlNmbhJnZvMXZylWY0lmc
vlmcw1ycu9WazNXZm9mcw1yclJHd1FWL5ETLklmdvNWLu9Wa0FmbpN2YhZ3L">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
Précédent
</span>
</div>
<div class="next dl-rounded-borders dl-white-bg">
<span class="disabled">
Suivant
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
</span>
</div>
</div>
"""
doc = html.document_fromstring(htmlString)

response = Response()
response._content = b'{}'

centers_page = CentersPage(browser=Browser(), response=response)
centers_page.doc = doc
next_page = centers_page.get_next_page()
assert next_page == None


@responses.activate
def test_get_next_page_de_should_return_2_on_page_1(tmp_path):
"""
Check that get_next_page returns 2 when we are on page 1 and next page is available
"""

htmlString = """
<div class="next-previous-links">
<div class="previous dl-rounded-borders dl-white-bg">
<span class="disabled">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
vorherige Seite
</span>
</div>
<div class="next dl-rounded-borders dl-white-bg">
<a href="/impfung-covid-19-corona/berlin?page=2&amp;ref_visit_motive_ids%5B%5D=6769">
Nächste Seite
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
</a>
</div>
</div>
"""
doc = html.document_fromstring(htmlString)

response = Response()
response._content = b'{}'

centers_page = CentersPage(browser=Browser(), response=response)
centers_page.doc = doc
next_page = centers_page.get_next_page()
assert next_page == 2


@responses.activate
def test_get_next_page_de_should_return_3_on_page_2(tmp_path):
"""
Check that get_next_page returns 3 when we are on page 2 and next page is available
"""

htmlString = """
<div class="next-previous-links">
<div class="previous dl-rounded-borders dl-white-bg">
<a href="/impfung-covid-19-corona/berlin?ref_visit_motive_ids%5B%5D=6769">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
vorherige Seite
</a>
</div>
<div class="next dl-rounded-borders dl-white-bg">
<a href="/impfung-covid-19-corona/berlin?page=3&amp;ref_visit_motive_ids%5B%5D=6769">
Nächste Seite
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
</a>
</div>
</div>
"""
doc = html.document_fromstring(htmlString)

response = Response()
response._content = b'{}'

centers_page = CentersPage(browser=Browser(), response=response)
centers_page.doc = doc
next_page = centers_page.get_next_page()
assert next_page == 3


@responses.activate
def test_get_next_page_de_should_return_4_on_page_3(tmp_path):
"""
Check that get_next_page returns 4 when we are on page 3 and next page is available
"""

htmlString = """
<div class="next-previous-links">
<div class="previous dl-rounded-borders dl-white-bg">
<a href="/impfung-covid-19-corona/berlin?page=2&amp;ref_visit_motive_ids%5B%5D=6769">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
vorherige Seite
</a>
</div>
<div class="next dl-rounded-borders dl-white-bg">
<a href="/impfung-covid-19-corona/berlin?page=4&amp;ref_visit_motive_ids%5B%5D=6769">
Nächste Seite
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
</a>
</div>
</div>
"""
doc = html.document_fromstring(htmlString)

response = Response()
response._content = b'{}'

centers_page = CentersPage(browser=Browser(), response=response)
centers_page.doc = doc
next_page = centers_page.get_next_page()
assert next_page == 4


def test_get_next_page_de_should_return_None_on_last_page(tmp_path):
"""
Check that get_next_page returns None when we are on the last page
"""

htmlString = """
<div class="next-previous-links">
<div class="previous dl-rounded-borders dl-white-bg">
<a href="/impfung-covid-19-corona/berlin?page=5&amp;ref_visit_motive_ids%5B%5D=6769">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
vorherige Seite
</a>
</div>
<div class="next dl-rounded-borders dl-white-bg">
<span class="disabled">
Nächste Seite
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
</span>
</div>
</div>
"""
doc = html.document_fromstring(htmlString)

response = Response()
response._content = b'{}'

centers_page = CentersPage(browser=Browser(), response=response)
centers_page.doc = doc
next_page = centers_page.get_next_page()
assert next_page == None


@responses.activate
def test_book_slots_should_succeed(tmp_path):
"""
Expand Down

0 comments on commit 7bd2004

Please sign in to comment.