start_crawl.py

import os
import logging
import schedule
import time
from time import strftime
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from webSpider.spiders.beijing import BATCM
from webSpider.spiders.hebei import HCOHP
from webSpider.spiders.shanxi import HCOSP
from webSpider.spiders.guojia import NATCM
from webSpider.spiders.shaanxi import SATCM
from webSpider.spiders.jilin import JLTCM
from webSpider.spiders.anhui import AHTCM

import argparse

parser = argparse.ArgumentParser(description="Crawl Parameters")
parser.add_argument(
    "-m",
    "--mode",
    help=' 爬虫模式：测试模式 or 生产模式 - "dev(development)" / "prod(production)"，默认 "prod" ',
    default="prod",
)
parser.add_argument(
    "-a",
    "--auto",
    help=' 是否开启定点自动爬虫 - "auto(Automatic)" / "non(Non-automatic)"，默认 "auto" ',
    default="auto",
)
args = parser.parse_args()

crawl_mode = args.mode
crawl_auto = args.auto


def job():
    check_folder = os.path.isdir("./logs")
    if not check_folder:
        os.makedirs("./logs/", mode=0o755)

    current_time = strftime("%Y-%m-%dT%H:%M:%S%z")
    logging.basicConfig(
        format="%(asctime)s %(levelname)s:%(message)s",
        filename=f"./logs/scrapy_{current_time}.log",
        level=logging.WARNING,
    )  # ISO 8601 Timestamp format

    process = CrawlerProcess(get_project_settings())

    process.crawl(BATCM, mode=crawl_mode)
    process.crawl(HCOHP, mode=crawl_mode)
    process.crawl(HCOSP, mode=crawl_mode)
    process.crawl(NATCM, mode=crawl_mode)
    process.crawl(SATCM, mode=crawl_mode)
    process.crawl(JLTCM, mode=crawl_mode)
    process.crawl(AHTCM, mode=crawl_mode)

    process.start()


if crawl_auto == "non":
    job()
else:
    schedule.every().day.at("03:35").do(job)

    logging.info("Start crawling.")

    while True:
        schedule.run_pending()
        time.sleep(1)