Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

4682 Updated cl_send_alerts to support V2 webhooks while maintaining compatibility with V1 webhooks #4705

Open
wants to merge 6 commits into
base: 4657-allow-users-to-select-webhooks-version
Choose a base branch
from
+403 −124
Open

4682 Updated cl_send_alerts to support V2 webhooks while maintaining compatibility with V1 webhooks #4705

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
feee7a2
feat(alerts): Added V2 webhook support to cl_send_alerts
albertisfu Nov 19, 2024
b580bea
fix(elasticsearch): Improved set_results_child_docs helper method
albertisfu Nov 22, 2024
61611ce
Merge branch '4657-allow-users-to-select-webhooks-version' into 4682-…
albertisfu Nov 22, 2024
a6b05f1
fix(elasticsearch): Solved merge conflicts
albertisfu Nov 22, 2024
d12e30b
fix(api): Fixed score merge and improved related test
albertisfu Nov 22, 2024
9c1b7cc
fix(api): Fixed people V4 API test to return scores
albertisfu Nov 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 85 additions & 25 deletions cl/alerts/management/commands/cl_send_alerts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,25 @@
from django.template import loader
from django.urls import reverse
from django.utils.timezone import now
from elasticsearch_dsl import MultiSearch
from elasticsearch_dsl import Q as ES_Q
from elasticsearch_dsl.response import Response

from cl.alerts.models import Alert, RealTimeQueue
from cl.alerts.utils import InvalidDateError
from cl.api.models import WebhookEventType
from cl.api.models import WebhookEventType, WebhookVersions
from cl.api.webhooks import send_search_alert_webhook
from cl.lib import search_utils
from cl.lib.command_utils import VerboseCommand, logger
from cl.lib.elasticsearch_utils import do_es_api_query
from cl.lib.elasticsearch_utils import (
do_es_api_query,
limit_inner_hits,
set_results_child_docs,
set_results_highlights,
)
from cl.lib.scorched_utils import ExtraSolrInterface
from cl.lib.search_utils import regroup_snippets
from cl.lib.types import CleanData
from cl.search.constants import ALERTS_HL_TAG, SEARCH_ALERTS_OPINION_HL_FIELDS
from cl.search.documents import OpinionDocument
from cl.search.forms import SearchForm
Expand Down Expand Up @@ -106,6 +114,59 @@ def send_alert(user_profile, hits):
msg.send(fail_silently=False)


def query_alerts_es(
cd: CleanData, v1_webhook: bool = False
) -> tuple[Response, Response | None]:
"""Query ES for opinion alerts, optionally handling a V1 webhook query.

:param cd: A CleanData object containing the query parameters.
:param v1_webhook: A boolean indicating whether to include a V1 webhook query.
:return: A tuple containing the main search response and an optional V1
query response.
"""

v1_results = None
search_query = OpinionDocument.search()
cd["highlight"] = True
main_query, _ = do_es_api_query(
search_query,
cd,
SEARCH_ALERTS_OPINION_HL_FIELDS,
ALERTS_HL_TAG,
"v4",
)
main_query = main_query.extra(
from_=0,
size=settings.SCHEDULED_ALERT_HITS_LIMIT,
)
multi_search = MultiSearch()
multi_search = multi_search.add(main_query)

if v1_webhook:
search_query = OpinionDocument.search()
v1_query, _ = do_es_api_query(
search_query,
cd,
SEARCH_ALERTS_OPINION_HL_FIELDS,
ALERTS_HL_TAG,
"v3",
)
v1_query = v1_query.extra(
from_=0,
size=settings.SCHEDULED_ALERT_HITS_LIMIT,
)
multi_search = multi_search.add(v1_query)

responses = multi_search.execute()
results = responses[0]
limit_inner_hits({}, results, cd["type"])
set_results_highlights(results, cd["type"])
set_results_child_docs(results)
if v1_webhook:
v1_results = responses[1]
return results, v1_results


class Command(VerboseCommand):
help = (
"Sends the alert emails on a real time, daily, weekly or monthly "
Expand Down Expand Up @@ -152,10 +213,9 @@ def handle(self, *args, **options):
if options["rate"] == Alert.REAL_TIME:
self.clean_rt_queue()

def run_query(self, alert, rate):
def run_query(self, alert, rate, v1_webhook=False):
results = []
cd = {}
main_params = {}
v1_results = None
logger.info(f"Now running the query: {alert.query}\n")

# Make a dict from the query string.
Expand All @@ -175,7 +235,7 @@ def run_query(self, alert, rate):
if waffle.switch_is_active("oa-es-alerts-active"):
# Return empty results for OA alerts. They are now handled
# by Elasticsearch.
return query_type, results
return query_type, results, v1_results

logger.info(f"Data sent to SearchForm is: {qd}\n")
search_form = SearchForm(qd, is_es_form=self.o_es_alerts)
Expand All @@ -187,7 +247,7 @@ def run_query(self, alert, rate):
and len(self.valid_ids[query_type]) == 0
):
# Bail out. No results will be found if no valid_ids.
return query_type, results
return query_type, results, v1_results

main_params = search_utils.build_main_query(
cd,
Expand Down Expand Up @@ -220,19 +280,7 @@ def run_query(self, alert, rate):
)

if self.o_es_alerts:
search_query = OpinionDocument.search()
s, _ = do_es_api_query(
search_query,
cd,
SEARCH_ALERTS_OPINION_HL_FIELDS,
ALERTS_HL_TAG,
"v3",
)
s = s.extra(
from_=0,
size=settings.SCHEDULED_ALERT_HITS_LIMIT,
)
results = s.execute()
results, v1_results = query_alerts_es(cd, v1_webhook)
else:
# Ignore warnings from this bit of code. Otherwise, it complains
# about the query URL being too long and having to POST it instead
Expand All @@ -248,7 +296,7 @@ def run_query(self, alert, rate):
regroup_snippets(results)

logger.info(f"There were {len(results)} results.")
return qd, results
return qd, results, v1_results

def send_emails_and_webhooks(self, rate):
"""Send out an email and webhook events to every user whose alert has a
Expand All @@ -261,14 +309,23 @@ def send_emails_and_webhooks(self, rate):
alerts = user.alerts.filter(rate=rate)
logger.info(f"Running alerts for user '{user}': {alerts}")

# Query user's webhooks.
user_webhooks = user.webhooks.filter(
event_type=WebhookEventType.SEARCH_ALERT, enabled=True
)
v1_webhook = WebhookVersions.v1 in {
webhook.version for webhook in user_webhooks
}
if rate == Alert.REAL_TIME:
if not user.profile.is_member:
continue

hits = []
for alert in alerts:
try:
qd, results = self.run_query(alert, rate)
qd, results, v1_results = self.run_query(
alert, rate, v1_webhook
)
except:
traceback.print_exc()
logger.info(
Expand All @@ -293,10 +350,13 @@ def send_emails_and_webhooks(self, rate):

# Send webhook event if the user has a SEARCH_ALERT
# endpoint enabled.
user_webhooks = user.webhooks.filter(
event_type=WebhookEventType.SEARCH_ALERT, enabled=True
)
for user_webhook in user_webhooks:
results = (
v1_results
if alert.alert_type == SEARCH_TYPES.OPINION
and user_webhook.version == WebhookVersions.v1
else results
)
send_search_alert_webhook(
self.sis[search_type], results, user_webhook, alert
)
Expand Down
67 changes: 51 additions & 16 deletions cl/alerts/templates/alert_email_es.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,26 +75,63 @@ <h3 class="alt bottom" style="font-size: 1.5em; font-weight: normal; line-height
</a>
<br>
{% endif %}
{% else %}
<p style="font-size: 100%; font-weight: inherit; font-family: inherit; border: 0; vertical-align: baseline; font-style: inherit; margin: 0; padding: 0;">
<strong style="font-weight: bold;">
View original:
</strong>
{% if result.download_url %}
<a href="{{result.download_url}}" style="font-size: 100%; font-weight: inherit; font-family: inherit; color: #009; border: 0; font-style: inherit; padding: 0; text-decoration: none; vertical-align: baseline; margin: 0;">
From the court
</a>
&nbsp;&nbsp;|&nbsp;&nbsp;
{% endif %}
{% endif %}
{% if type == 'o' %}
<ul>
{% for doc in result.child_docs %}
{% with doc=doc|get_es_doc_content:True %}
<li style="margin-bottom: 5px;">
{% if result.child_docs|length > 1 or doc.type != 'combined-opinion' %}
{% if doc.text %}
<strong>{{ doc.type_text }}</strong>
{% endif %}
{% endif %}
{% if doc.text %}
{% contains_highlights doc.text.0 True as highlighted %}
<span style="display: block;">{% if highlighted %}&hellip; {% endif %}{{ doc.text|render_string_or_list|safe|underscore_to_space }} &hellip;</span>
{% endif %}
{% if doc.download_url or doc.local_path %}
<p style="font-size: 100%; font-weight: inherit; font-family: inherit; border: 0; vertical-align: baseline; font-style: inherit; margin: 0; padding: 0;">
<strong style="font-weight: bold;">
View original:
</strong>
{% if doc.download_url %}
<a href="{{doc.download_url}}" style="font-size: 100%; font-weight: inherit; font-family: inherit; color: #009; border: 0; font-style: inherit; padding: 0; text-decoration: none; vertical-align: baseline; margin: 0;">
From the court
</a>
&nbsp;&nbsp;|&nbsp;&nbsp;
{% endif %}
{% if doc.local_path %}
{# Provide link to S3. #}
<a href="https://storage.courtlistener.com/{{doc.local_path}}" style="font-size: 100%; font-weight: inherit; font-family: inherit; color: #009; border: 0; font-style: inherit; padding: 0; text-decoration: none; vertical-align: baseline; margin: 0;">
Our backup
</a>
{% endif %}
</p>
{% endif %}
</li>
{% endwith %}
{% endfor %}
</ul>
{% endif %}
{% if type == 'oa' %}
<p style="font-size: 100%; font-weight: inherit; font-family: inherit; border: 0; vertical-align: baseline; font-style: inherit; margin: 0; padding: 0;">
<strong style="font-weight: bold;">
View original:
</strong>
{% if result.download_url %}
<a href="{{result.download_url}}" style="font-size: 100%; font-weight: inherit; font-family: inherit; color: #009; border: 0; font-style: inherit; padding: 0; text-decoration: none; vertical-align: baseline; margin: 0;">
From the court
</a>
&nbsp;&nbsp;|&nbsp;&nbsp;
{% endif %}
{% if result.local_path %}
{# Provide link to S3. #}
<a href="https://storage.courtlistener.com/{{result.local_path}}" style="font-size: 100%; font-weight: inherit; font-family: inherit; color: #009; border: 0; font-style: inherit; padding: 0; text-decoration: none; vertical-align: baseline; margin: 0;">
Our backup
</a>
{% endif %}
</p>
{% endif %}
{% if type == 'oa' %}
<p style="font-size: 100%; font-weight: inherit; font-family: inherit; border: 0; vertical-align: baseline; font-style: inherit; margin: 0 0 0; padding: 0;">
<strong style="font-weight: bold;">Date Argued: </strong>
{% if result.dateArgued %}
Expand All @@ -116,9 +153,7 @@ <h3 class="alt bottom" style="font-size: 1.5em; font-weight: normal; line-height
{{ result|get_highlight:"judge"|safe|underscore_to_space }}
{% endif %}
</p>
{% endif %}
{% if type == 'o' or type == 'oa' %}
<p style="font-size: 100%; font-weight: inherit; font-family: inherit; border: 0; vertical-align: baseline; font-style: inherit; margin: 0 0 1.5em; padding: 0;">
<p style="font-size: 100%; font-weight: inherit; font-family: inherit; border: 0; vertical-align: baseline; font-style: inherit; margin: 0 0 1.5em; padding: 0;">
{% if result|get_highlight:"text" %}
&hellip;{{ result|get_highlight:"text"|safe|underscore_to_space }}&hellip;
{% endif %}
Expand Down
15 changes: 10 additions & 5 deletions cl/alerts/templates/alert_email_es.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,14 @@ View Full Results / Edit this Alert: https://www.courtlistener.com/?{{ alert.que
Disable this Alert (one click): https://www.courtlistener.com{% url "disable_alert" alert.secret_key %}{% endif %}

{{forloop.counter}}. {{ result.caseName|render_string_or_list|safe|striptags }} ({% if result.court_id != 'scotus' %}{{ result.court_citation_string|render_string_or_list|striptags }} {% endif %}{% if type == 'o' or type == 'r' %}{{ result.dateFiled|date:"Y" }}{% elif type == 'oa' %}{{ result.dateArgued|date:"Y" }}{% endif %})
{% if type == 'oa' %}{% if result.dateArgued %}Date Argued: {{ result.dateArgued|date:"F jS, Y" }}{% else %}Date Argued: Unknown Date {% endif %}{% if result.docketNumber %} | Docket Number: {{ result.docketNumber|render_string_or_list|safe|striptags }}{% endif %} | Duration: {{ result.duration|naturalduration }}{% if result.judge %} | Judge: {{ result.judge|render_string_or_list|safe|striptags|underscore_to_space }}{% endif %}{% endif %}
{% if type == 'o' or type == 'oa' %}{% if result|get_highlight:"text" %}...{{ result|get_highlight:"text"|safe|striptags|underscore_to_space|compress_whitespace }}...{% endif %}{% endif %}
{% if type == 'oa' %}{% if result.dateArgued %}Date Argued: {{ result.dateArgued|date:"F jS, Y" }}{% else %}Date Argued: Unknown Date {% endif %}{% if result.docketNumber %} | Docket Number: {{ result.docketNumber|render_string_or_list|safe|striptags }}{% endif %} | Duration: {{ result.duration|naturalduration }}{% if result.judge %} | Judge: {{ result.judge|render_string_or_list|safe|striptags|underscore_to_space }}{% endif %}
{% if result|get_highlight:"text" %}...{{ result|get_highlight:"text"|safe|striptags|underscore_to_space|compress_whitespace }}...{% endif %}
{% endif %}
{% if type == 'o' %}{% for doc in result.child_docs %}{% with doc=doc|get_es_doc_content:True %}{% if result.child_docs|length > 1 or doc.type != 'combined-opinion' %}{% if doc.text %}{{ doc.type_text }}{% endif %}{% endif %}
{% if doc.text %}...{{ doc.text|render_string_or_list|safe|striptags|underscore_to_space|compress_whitespace }}...{% endif %}
{% if doc.download_url %} - Download original from the court: {{doc.download_url}}{% endif %}
{% if doc.local_path %} - Download the original from our backup: https://storage.courtlistener.com/{{ doc.local_path }}{% endif %}
{% endwith %}{% endfor %}{% endif %}
{% if type == 'r' %}{% if result.dateFiled %}Date Filed: {{ result.dateFiled|date:"F jS, Y" }}{% else %}Date Filed: Unknown Date {% endif %}{% if result.docketNumber %} | Docket Number: {{ result.docketNumber|render_string_or_list|safe|striptags }}{% endif %}
{% for doc in result.child_docs %}{% with doc=doc|get_es_doc_content:scheduled_alert %} - {% if doc.short_description %}{{ doc.short_description|render_string_or_list|safe|striptags }} - {% endif %}Document #{% if doc.document_number %}{{ doc.document_number }}{% endif %}{% if doc.attachment_number %}, Attachment #{{ doc.attachment_number }}{% endif %}
{% if doc.description %}Description: {{ doc.description|render_string_or_list|safe|striptags }}{% endif %}
Expand All @@ -27,9 +33,8 @@ Disable this Alert (one click): https://www.courtlistener.com{% url "disable_ale
{% if result.child_docs and result.child_remaining %}{% extract_q_value alert.query_run as q_value %}View Additional Results for this Case: https://www.courtlistener.com/?type={{ type|urlencode }}&q={% if q_value %}({{ q_value|urlencode }})%20AND%20{% endif %}docket_id%3A{{ result.docket_id|urlencode }}{% endif %}
{% endif %}~~~~~
- View this item on our site: https://www.courtlistener.com{% if type == 'r' %}{{result.docket_absolute_url}}{% else %}{{result.absolute_url}}{% endif %}
{% if result.download_url %} - Download original from the court: {{result.download_url}}
{% endif %}{% if result.local_path %} - Download the original from our backup: https://storage.courtlistener.com/{{ result.local_path }}{% endif %}{% endfor %}

{% if type == 'oa' %}{% if result.download_url %} - Download original from the court: {{result.download_url}}
{% endif %}{% if result.local_path %} - Download the original from our backup: https://storage.courtlistener.com/{{ result.local_path }}{% endif %}{% endif %}{% endfor %}
{% endfor %}
************************
This alert brought to you by the 501(c)(3) non-profit Free Law Project
Expand Down
Loading