Пауза между объявлениями

.github/workflows/deploy.yml

@@ -0,0 +1,40 @@
+on:
+  push:
+    branches: [ "main" ]
+
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Executing remote ssh command
+        uses: appleboy/ssh-action@master
+        with:
+          host: ${{ secrets.HOST }}
+          port: ${{ secrets.PORT }}
+          username: ${{ secrets.USER }}
+          key: ${{ secrets.SSH }}
+          script: |
+            echo "Stoping old version"
+            if [ -d ~/parser_avito ];
+            then
+                cd ~/parser_avito
+                make stop
+            fi
+
+            echo "Removing old version"
+            cd ~
+            rm -rf parser_avito
+            # yes | sudo docker system prune -a
+
+            echo "Pulling new version"
+            GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" git clone [email protected]:altservices/parser_avito.git
+
+            # echo "Copying keys"
+            # cp ~/.secrets/parser_avito.env ~/parser_avito/.env
+
+            echo "Starting new version"
+            cd ~/parser_avito
+            make run

.gitignore

@@ -0,0 +1,12 @@
+# Cache
+.DS_Store
+__pycache__/
+
+# Environment
+venv/
+env/
+
+# Data
+downloaded_files/
+result/
+viewed.txt

Dockerfile

@@ -0,0 +1,44 @@
+# Pull base image
+FROM python:3.10
+
+# Set work directory
+WORKDIR /app
+
+# Установка зависимостей для Selenium и Chrome
+RUN apt update
+RUN apt install -y \
+    wget \
+    xvfb \
+    unzip \
+    libxi6 \
+    libgconf-2-4 \
+    libnss3
+
+# Установка Chrome
+RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+RUN dpkg -i google-chrome-stable_current_amd64.deb; apt-get -fy install
+
+# # Скачивание и распаковка Chrome
+# RUN wget -q https://storage.googleapis.com/chrome-for-testing-public/122.0.6261.69/linux64/chrome-linux64.zip \
+#     && unzip chrome-linux64.zip -d /opt/ \
+#     && ln -s /opt/chrome-linux/chrome /usr/bin/google-chrome \
+#     && rm chrome-linux64.zip
+
+# # Установка ChromeDriver
+# RUN wget https://storage.googleapis.com/chrome-for-testing-public/122.0.6261.69/linux64/chromedriver-linux64.zip
+# RUN unzip chromedriver-linux64.zip -d /usr/bin/
+# # RUN mv chromedriver /usr/bin/chromedriver
+# # RUN chmod +x /usr/bin/chromedriver
+
+RUN apt-get update && apt-get install -y chromium-driver
+RUN chmod +x /usr/bin/chromedriver
+
+# Copy project
+COPY . /app
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+# Install dependencies
+RUN pip install -r requirements.txt

Makefile

@@ -0,0 +1,26 @@
+dev:
+	docker compose -p parser_avito up --build
+
+run:
+	docker compose -f compose.prod.yml -p parser_avito up --build -d
+
+stop:
+	docker compose -f compose.prod.yml -p parser_avito stop
+
+check:
+	docker ps --filter name="^parser_avito" --format "table {{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}"
+
+connect:
+	docker exec -it `docker ps -a | grep parser_avito-script | cut -d ' ' -f 1` bash
+
+logs:
+	docker logs `docker ps -a | grep parser_avito-script | cut -d ' ' -f 1`
+
+set:
+	cp nginx.prod.conf /etc/nginx/sites-enabled/alt.conf
+	sudo chmod 0755 ~
+	sudo chmod -R a+w ~/parser_avito/result/
+	sudo chmod 0700 ~/.ssh
+	sudo chmod -R 0600 ~/.ssh/*
+	sudo systemctl restart nginx
+	sudo certbot --nginx

compose.prod.yml

@@ -0,0 +1,40 @@
+version: "3.9"
+services:
+  script:
+    image: parser_avito/script
+    build: .
+    restart: unless-stopped
+    volumes:
+      - /root/parser_avito/result:/app/result
+    command: bash -c "cd /app && python parser_cls.py"
+
+#   jitsu:
+#     image: jitsucom/server:latest
+#     ports:
+#       - "8000:8001"
+#     depends_on:
+#       - db
+#     environment:
+#       DATABASE_URL: postgres://jitsu:asdr1234@db/jitsu?sslmode=disable
+#       CONFIGURATOR_SECRET: asdr1234
+#     volumes:
+#       - ./jitsu.conf:/home/jitsu/config
+#       - /root/data/jitsu:/var/lib/jitsu/data
+
+#   db:
+#     image: postgres:13
+#     ports:
+#       - "5432:5432"
+#     environment:
+#       POSTGRES_USER: jitsu
+#       POSTGRES_PASSWORD: asdr1234
+#       POSTGRES_DB: jitsu
+#     volumes:
+#       - ./postgresql.conf:/etc/postgresql/postgresql.conf
+#       - ./pg_hba.conf:/etc/postgresql/pg_hba.conf
+#       - /root/data/pg:/var/lib/postgresql/data
+
+# volumes:
+#   jitsu-config:
+#   jitsu-data:
+#   postgres-data:

jitsu.conf

@@ -0,0 +1,37 @@
+server:
+  log_level: info  # Adjust logging level as needed (debug, info, warning, error)
+
+database:
+  type: postgres
+  postgres:
+    host: db  # Matches the service name in docker-compose
+    port: 5432  # Default PostgreSQL port
+    dbname: jitsu  # Database name as specified in docker-compose environment
+    schema: public
+    user: jitsu  # Matches the user specified in docker-compose environment
+    password: asdr1234  # Ensure this matches what you've set in docker-compose
+    sslmode: disable  # Adjust according to your SSL setup
+
+# destinations:
+#   bigquery_example:  # Destination name
+#     type: bigquery  # Destination type (bigquery, postgres, clickhouse, etc.)
+#     mode: stream  # Insert mode (stream or batch)
+#     data_layout:
+#       table_name_template: events_{{.EventName}}  # Naming pattern for tables
+#     google:
+#       project_id: your_project_id  # Your Google Cloud project ID
+#       json_key: your_json_key  # Your JSON key for authentication
+
+#   postgres_example:
+#     type: postgres
+#     mode: batch
+#     data_layout:
+#       table_name_template: events_{{.EventName}}
+#     postgres:
+#       host: localhost  # Or another host where your PostgreSQL is
+#       port: 5432
+#       dbname: your_dbname
+#       user: your_user
+#       password: your_password
+#       parameters:  # Any additional connection parameters
+#         sslmode: disable

lib/req.py

@@ -0,0 +1,40 @@
+import aiohttp
+
+
+async def fetch(
+    url,
+    payload=None,
+    type_req="post",
+    type_data="json",
+    headers=None,
+):
+    if payload is None:
+        payload = {}
+
+    async with aiohttp.ClientSession() as session:
+        if type_req == "post":
+            req = session.post
+        elif type_req == "put":
+            req = session.put
+        elif type_req == "delete":
+            req = session.delete
+        elif type_req == "patch":
+            req = session.patch
+        elif type_req == "options":
+            req = session.options
+        else:
+            req = session.get
+
+        async with req(
+            url,
+            headers=headers,
+            **{type_data: payload},
+        ) as response:
+            code = response.status
+
+            try:
+                data = await response.json()
+            except:  # noqa: E722
+                data = await response.text()
+
+            return code, data

nginx.prod.conf

@@ -0,0 +1,11 @@
+server {
+    listen 80; # TODO: add http2 after certbot
+    server_name alt.chill.services;
+
+    root /;
+
+    location /data/parser/ {
+        alias /root/parser_avito/result/;
+        add_header Access-Control-Allow-Origin "*";
+    }
+}