fix json double quotes problem in BATCM

bucm-policy-search · Aug 27, 2021 · bef7c70 · bef7c70
1 parent 80a506d
commit bef7c70
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 4 deletions.
diff --git a/webSpider/pipelines.py b/webSpider/pipelines.py
@@ -64,8 +64,6 @@ def process_item(self, item, spider):
                 "query": {
                     "bool": {
                         "must": [
-                            {"term": {"title.keyword": {"value": item["title"]}}},
-                            {"match": {"publishingDate": item["publishingDate"]}},
                             {
                                 "term": {
                                     "urlSource.keyword": {"value": item["urlSource"]}

diff --git a/webSpider/spiders/BATCM.py b/webSpider/spiders/BATCM.py
@@ -134,7 +134,7 @@ def detailPage(self, response):
 
         # delete "\n" and spaces in title
         title_new = title_origin.strip(" \n")
-        item["title"] = re.sub(r"\s", "", title_new)
+        item["title"] = re.sub(r'"', "\u0022", title_new)
 
         date_origin = response.css("div.zhengwen div::text").get()
 
@@ -156,9 +156,10 @@ def detailPage(self, response):
         if ul != []:
             for li in ul:
                 mark = li.css("a::text").get()
+                new_mark = re.sub(r'"', "\u0022", mark)
 
                 link = response.urljoin(li.css("a::attr(href)").get())
-                attachment.append({"mark": mark, "link": link})
+                attachment.append({"mark": new_mark, "link": link})
         item["attachment"] = attachment
 
         yield item