Handling special cases in commentary with new field

app/crud/contents_crud.py

@@ -5,6 +5,7 @@
 import re
 from datetime import datetime
 from pytz import timezone
+from sqlalchemy import or_
 from sqlalchemy.orm import Session, defer, joinedload
 from sqlalchemy.sql import text
 import db_models #pylint: disable=import-error
@@ -16,13 +17,14 @@
 
 ist_timezone = timezone("Asia/Kolkata")
 
-def get_commentaries(db_: Session,**kwargs):
+def get_commentaries(db_: Session,**kwargs):#pylint: disable=too-many-locals
     '''Fetches rows of commentries from the table specified by resource_name'''
     resource_name = kwargs.get("resource_name")
     reference = kwargs.get("reference",None)
     commentary_id = kwargs.get("commentary_id",None)
     search_word = kwargs.get("search_word",None)
     commentary = kwargs.get("commentary",None)
+    section_type = kwargs.get("section_type",None)
     active = kwargs.get("active",True)
     skip = kwargs.get("skip",0)
     limit = kwargs.get("limit",100)
@@ -37,6 +39,10 @@ def get_commentaries(db_: Session,**kwargs):
     if commentary:
         query = query.filter(model_cls.commentary.contains(\
             utils.normalize_unicode(commentary.strip())))
+    if section_type:
+        filter_conditions = [model_cls.sectionType.contains([item]) for item in section_type]
+        filter_condition = or_(*filter_conditions)
+        query = query.filter(filter_condition)
     if reference:
         if isinstance(reference, str):
             reference = json.loads(reference)
@@ -142,10 +148,12 @@ def upload_commentaries(db_: Session, resource_name, commentaries, job_id, user_
             refStart=ref_start,
             refEnd=ref_end,
             commentary = utils.normalize_unicode(item.commentary),
+            sectionType = item.sectionType,
             active=item.active)
         row_out = {
             "reference" : ref,
             "commentary" :  utils.normalize_unicode(item.commentary),
+            "sectionType" : item.sectionType,
             "active": item.active}
         db_content.append(row)
         db_content_out.append(row_out)
@@ -203,12 +211,15 @@ def update_commentaries(db_: Session, resource_name, commentaries,job_id, user_i
             row.refEnd=ref_end
         if item.commentary:
             row.commentary = utils.normalize_unicode(item.commentary)
+        if item.sectionType:
+            row.sectionType = item.sectionType
         if item.active is not None:
             row.active = item.active
         db_.flush()
         db_content.append(row)
         row_out = {
             "reference" : row.reference,
+            "sectionType":row.sectionType,
             "commentary" :  utils.normalize_unicode(row.commentary),
             "active": row.active}
         db_content_out.append(row_out)

app/db_models.py

@@ -2,7 +2,8 @@
 
 from enum import Enum
 from datetime import datetime
-from sqlalchemy import Column, Integer, String, JSON, ARRAY, Float, text
+from sqlalchemy.dialects.postgresql import ARRAY
+from sqlalchemy import Column, Integer, String, JSON, Float, text
 from sqlalchemy import Boolean, ForeignKey, DateTime
 from sqlalchemy import UniqueConstraint, Index
 from sqlalchemy.sql import func
@@ -151,6 +152,7 @@ def ref_string(cls):# pylint: disable=E0213
     refStart = Column('ref_start', Integer)
     refEnd = Column('ref_end', Integer)
     commentary = Column('commentary', String)
+    sectionType = Column('section_type', ARRAY(String))
     active = Column('active', Boolean)
     __table_args__ = (
         {'extend_existing': True}

app/routers/content_apis.py

@@ -663,6 +663,7 @@ async def get_bible_verse(request: Request,
 @get_auth_access_check_decorator
 async def get_commentary(request: Request,
     resource_name: schemas.TableNamePattern=Path(..., examples="en_BBC_1_commentary"),
+    section_type: List[str] = Query(None, examples=["commentary-text", "book-introduction"]),
     reference: str = Query(None,
     examples='{"book": "mat", "chapter": 1, "verseNumber": 6}'),
     search_word: str=Query(None, examples="customary") ,
@@ -677,14 +678,18 @@ async def get_commentary(request: Request,
     * Value 0 for verse and last_verse indicate chapter introduction and -1 indicate
     chapter epilogue.
     * Similarly 0 for chapter means book introduction and -1 for chapter means book epilogue
+    * Available values of section_type : ["commentary-text"], ["book-introduction"], ["epilogue"],
+    ["chapter-introduction], ["topic-outline"] and Default value :[ "commentary-text" ]
     * skip=n: skips the first n objects in return list
     * limit=n: limits the no. of items to be returned to n
     * returns [] for not available content'''
     log.info('In get_commentary')
     log.debug('resource_name: %s, reference: %s, skip: %s, limit: %s, search_word: %s,\
-        commentary: %s', resource_name, reference, skip, limit,search_word,commentary)
-    return contents_crud.get_commentaries(db_, resource_name=resource_name,reference = reference,\
-        search_word = search_word, commentary=commentary, active=active, skip = skip, limit = limit)
+        commentary: %s,sectiopn_type: %s', resource_name, reference, skip,
+        limit,search_word,commentary,section_type)
+    return contents_crud.get_commentaries(db_, resource_name=resource_name,reference = reference,
+        search_word = search_word, commentary=commentary, section_type = section_type,
+        active=active, skip = skip, limit = limit)
 
 @router.post('/v2/resources/commentaries/{resource_name}',
     response_model=schema_content.CommentaryCreateResponse, response_model_exclude_none=True,
@@ -708,7 +713,9 @@ async def add_commentary(request: Request,background_tasks: BackgroundTasks,
     * Value 0 for verse and last_verse indicate chapter introduction and
     -1 indicate chapter epilogue.
     * Similarly 0 for chapter means book introduction and -1 for chapter means book epilogue,
-    verses fields can be null in these cases'''
+    verses fields can be null in these cases
+    * Available values of section_type : ["commentary-text"], ["book-introduction"], ["epilogue"],
+    ["chapter-introduction], ["topic-outline"] and Default value :[ "commentary-text" ]'''
     log.info('In add_commentary')
     log.debug('resource_name: %s, commentaries: %s',resource_name, commentaries)
     # verify resource exist

app/schema/schema_content.py

@@ -8,6 +8,7 @@
 from crud import utils
 
 #pylint: disable=too-few-public-methods
+
 class BibleBook(BaseModel):
     '''response object of Bible book'''
     bookId : int
@@ -224,13 +225,15 @@ class BookContentType(str, Enum):
 
 class CommentaryCreate(BaseModel):
     '''Response object for commentaries'''
+    sectionType: List[str] = ["commentary-text"]
     reference: Reference = None
     commentary: str
     active: bool = True
     class Config:
         '''display example value in API documentation'''
         schema_extra = {
             "example": {
+                "sectionType": ["commentary-text"],
                 "reference": {"book":"MRK", "chapter":11, "verseNumber":12,
                                "bookEnd":"LUK", "chapterEnd":14, "verseEnd":15 },
                 "commentary": "It was customary at the time ...",
@@ -242,6 +245,7 @@ class Config:
 class CommentaryEdit(BaseModel):
     '''Response object for commentaries'''
     commentaryId:int
+    sectionType: List[str] = ["commentary-text"]
     reference: Reference = None
     commentary: str = None
     active: bool = None
@@ -250,6 +254,7 @@ class Config:
         schema_extra = {
             "example": {
                 "commentaryId":100000,
+                "sectionType": ["commentary-text"],
                 "reference": {"book":"MRK", "chapter":15, "verseNumber":10,
                                "bookEnd":"JHN", "chapterEnd":1, "verseEnd":9 },
                 "commentary": "One of the practices of that time was ...",
@@ -260,6 +265,7 @@ class Config:
 class CommentaryResponse(BaseModel):
     '''Response object for commentaries'''
     commentaryId: int
+    sectionType: List[str]
     reference: Reference = None
     commentary: str
     active: bool
@@ -270,6 +276,7 @@ class Config:
         schema_extra = {
             "example": {
                 "commentaryId":100000,
+                "sectionType": ["commentary-text"],
                 "reference": {"book":"MRK", "chapter":11, "verseNumber":12,
                                "bookEnd":"LUK", "chapterEnd":14, "verseEnd":15 },
                 "commentary": "It was customary at the time ...",

app/test/test_commentaries.py

@@ -188,25 +188,25 @@ def test_post_incorrect_data():
 def test_get_after_data_upload(): #pylint: disable=too-many-statements
     '''Add some data into the table and do all get tests'''
     data = [
-        {'reference': {"book":"gen", "chapter":0},'commentary':'Book intro to Genesis'},
-        {'reference': {"book":"gen", "chapter":1, "verseNumber":0,"bookEnd":"gen","chapterEnd":1,"verseEnd":0},
+        {'sectionType':["book-introduction"],'reference': {"book":"gen", "chapter":0},'commentary':'Book intro to Genesis'},
+        {'sectionType':["commentary-text"],'reference': {"book":"gen", "chapter":1, "verseNumber":0,"bookEnd":"gen","chapterEnd":1,"verseEnd":0},
             'commentary':'chapter intro to Genesis 1'},
-        {'reference': {"book":"gen", "chapter":1, "verseNumber":1,"bookEnd":"gen","chapterEnd":1,"verseEnd":10},
+        {'sectionType':["commentary-text"],'reference': {"book":"gen", "chapter":1, "verseNumber":1,"bookEnd":"gen","chapterEnd":1,"verseEnd":10},
             'commentary':'the begining'},
-        {'reference': {"book":"gen", "chapter":1, "verseNumber":3,"bookEnd":"gen","chapterEnd":1,"verseEnd":30},
+        {'sectionType':["commentary-text"],'reference': {"book":"gen", "chapter":1, "verseNumber":3,"bookEnd":"gen","chapterEnd":1,"verseEnd":30},
             'commentary':'the creation'},
-        {'reference': {"book":"gen", "chapter":1, "verseNumber":-1,"bookEnd":"gen","chapterEnd":1,"verseEnd":-1},
+        {'sectionType':["epilogue"],'reference': {"book":"gen", "chapter":1, "verseNumber":-1,"bookEnd":"gen","chapterEnd":1,"verseEnd":-1},
             'commentary':'Chapter Epilogue. God completes creation in 6 days.'},
-        {'reference': {"book":"gen", "chapter":-1},'commentary':'book Epilogue.'},
-        {'reference': {"book":"exo", "chapter":1, "verseNumber":1,"bookEnd":"exo","chapterEnd":1,"verseEnd":1},
+        {'sectionType':["epilogue"],'reference': {"book":"gen", "chapter":-1},'commentary':'book Epilogue.'},
+        {'sectionType':["commentary-text"],'reference': {"book":"exo", "chapter":1, "verseNumber":1,"bookEnd":"exo","chapterEnd":1,"verseEnd":1},
             'commentary':'first verse of Exodus'},
-        {'reference': {"book":"exo", "chapter":1, "verseNumber":1,"bookEnd":"exo","chapterEnd":1,"verseEnd":10},
+        {'sectionType':["commentary-text"],'reference': {"book":"exo", "chapter":1, "verseNumber":1,"bookEnd":"exo","chapterEnd":1,"verseEnd":10},
             'commentary':'first para of Exodus'},
-        {'reference': {"book":"exo", "chapter":1, "verseNumber":1,"bookEnd":"exo","chapterEnd":1,"verseEnd":25},
+        {'sectionType':["commentary-text"],'reference': {"book":"exo", "chapter":1, "verseNumber":1,"bookEnd":"exo","chapterEnd":1,"verseEnd":25},
             'commentary':'first few paras of Exodus'},
         {'reference': {"book":"exo", "chapter":1, "verseNumber":20,"bookEnd":"est","chapterEnd":1,"verseEnd":25},
             'commentary':'a middle para of Exodus'},
-        {'reference': {"book":"exo", "chapter":0},'commentary':'Book intro to Exodus'}
+        {'sectionType':["book-introduction"],'reference': {"book":"exo", "chapter":0},'commentary':'Book intro to Exodus'}
     ]
 
     resp, resource_name = check_post(data)
@@ -218,6 +218,7 @@ def test_get_after_data_upload(): #pylint: disable=too-many-statements
     assert 'output' in job_response.json()['data']
     assert len(data) == len(job_response.json()['data']['output']['data'])
     for item in job_response.json()['data']['output']['data']:
+        # print("///item:",item)
         assert_positive_get(item)
     headers_auth['Authorization'] = "Bearer"+" "+initial_test_users['VachanAdmin']['token']
     check_default_get(UNIT_URL+resource_name, headers_auth,assert_positive_get)
@@ -229,17 +230,31 @@ def test_get_after_data_upload(): #pylint: disable=too-many-statements
     assert response.json()["error"] == "Authentication Error"
 
     #with auth
-    response = client.get(UNIT_URL+resource_name+'?commentary=Book intro to Genesis',\
+    response = client.get(UNIT_URL+resource_name+'?commentary=Book intro to Genesis&section_type=book-introduction',\
         headers=headers_auth)
     assert response.status_code == 200
     assert len(response.json()) == 1
 
+    #searching multiple section types
+    response = client.get(UNIT_URL+resource_name+'?section_type=book-introduction&section_type=epilogue',\
+        headers=headers_auth)
+    assert response.status_code == 200
+    assert len(response.json()) == 4
+
     response = client.get(UNIT_URL+resource_name+"?search_word=gen",headers=headers_auth)
     assert response.status_code == 200
     assert len(response.json()) == 6
 
+    response = client.get(UNIT_URL+resource_name+"?search_word=gen&section_type=book-introduction",headers=headers_auth)
+    assert response.status_code == 200
+    assert len(response.json()) == 1
+
+    response = client.get(UNIT_URL+resource_name+"?search_word=gen&section_type=epilogue",headers=headers_auth)
+    assert response.status_code == 200
+    assert len(response.json()) == 2
+
     # all book introductions
-    response = client.get(UNIT_URL+resource_name+"?commentary=Book intro",headers=headers_auth)
+    response = client.get(UNIT_URL+resource_name+"?commentary=Book intro&section_type=book-introduction",headers=headers_auth)
     assert response.status_code == 200
     assert len(response.json()) == 2
 
@@ -309,6 +324,17 @@ def test_get_after_data_upload(): #pylint: disable=too-many-statements
         "verseNumber":9,"bookEnd":"rev","chapterEnd":6,"verseEnd":4}', headers=headers_auth)
     assert_not_available_content(response)
 
+def test_post_incorrect_sectiontype_format():
+    '''Check input validations for section type field'''
+
+    #Posting section type as a non-list field
+    data = [
+        {'sectionType':"book-introduction",'reference': {"book":"jhn", "chapter":0},'commentary':'Book intro to John'}
+        ]
+    resp= check_post(data)[0]
+    print("////:",resp.json()["details"],resp.status_code)
+    assert resp.status_code == 422
+
 def test_get_incorrect_data():
     '''Check for input validations in get'''
     # Post data
@@ -345,15 +371,18 @@ def test_put_after_upload():
     assert len(data) == len(job_response.json()['data']['output']['data'])
     for item in job_response.json()['data']['output']['data']:
         assert_positive_get(item)
+    headers_auth['Authorization'] = "Bearer"+" "+initial_test_users['VachanAdmin']['token']
     get_response = client.get(UNIT_URL+resource_name,headers=headers_auth)
     commentary_id1 = get_response.json()[0]['commentaryId']
     commentary_id2 = get_response.json()[1]['commentaryId']
 
     # positive PUT
     new_data = [
-        {'commentaryId':commentary_id1,'reference': {"book":"mat", "chapter":1,'verseNumber':1,'bookEnd':"mat","chapterEnd":1,'verseEnd':10},
+        {'commentaryId':commentary_id1,'sectionType':["commentary-text"],
+            'reference': {"book":"mat", "chapter":1,'verseNumber':1,'bookEnd':"mat","chapterEnd":1,'verseEnd':10},
             'commentary':"first verses of matthew"},
-        {'commentaryId':commentary_id2,'reference': {"book":"mrk", "chapter":0,"bookEnd":"mrk","chapterEnd":0,'verseEnd':0},
+        {'commentaryId':commentary_id2,'sectionType':["book-introduction"],
+            'reference': {"book":"mrk", "chapter":0,"bookEnd":"mrk","chapterEnd":0,'verseEnd':0},
             'commentary':"book intro to Mark"}
     ]
     #without auth
@@ -370,6 +399,7 @@ def test_put_after_upload():
     assert job_response.json()["message"] == "Commentaries updated successfully"
     for i,item in enumerate(job_response.json()['data']['output']['data']):
         assert_positive_get(item)
+        assert item['sectionType'] == new_data[i]['sectionType']
         assert item['commentary'] == new_data[i]['commentary']
         assert item['reference'] == new_data[i]['reference']