Merge pull request #3 from AlwaysBeCalm/main

completed the pipeline using github actions

.github/workflows/updated-json-files.yml

@@ -0,0 +1,37 @@
+name: Generate needed formats from JSON
+on:
+  push:
+    paths:
+      - 'companies-support-isreal.json'
+      - 'israel-companies-services.json'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: install required libraries
+        run: pip install -r requirements.txt
+
+      - name: run the automated tasks
+        run: python3 -B util/automated-tasks.py
+
+      - name: add and commit generated file
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Actions"
+          
+          git add .
+          git commit -m "automatically converted after editing companies-support-isreal.json and/or israel-companies-services.json"
+
+      - name: push the generated file
+        uses: ad-m/[email protected]
+        with:
+          github_token: ${{ secrets.GIT_ACTION_TOKEN }}
+          branch: main

.gitignore

@@ -0,0 +1 @@
+venv/

israel-companies-services.json

@@ -1,4 +1,5 @@
 {
+
 	"companiesAndServices": [
 		{
 			"id": 1,

requirements.txt

@@ -0,0 +1,18 @@
+attrs==22.1.0
+certifi==2022.9.24
+charset-normalizer==2.1.1
+et-xmlfile==1.1.0
+idna==3.4
+iniconfig==1.1.1
+openpyxl==3.0.10
+packaging==21.3
+pluggy==1.0.0
+py==1.11.0
+PyArabic==0.6.15
+pyparsing==3.0.9
+pytest==7.1.3
+PyYAML==6.0
+requests==2.28.1
+six==1.16.0
+tomli==2.0.1
+urllib3==1.26.12

util/automated-tasks.py

@@ -0,0 +1,128 @@
+import json, glob, os
+
+################################
+# Define Functions
+################################
+
+
+def read_json_file(file_path: str) -> dict:
+    """
+    read the content of json file, and return its content
+    :param file_path: the path of the json file.
+    :returns: dictionary of the json data.
+    """
+    with open(file_path) as fr:
+        return json.load(fr)
+
+
+def write_json_file(file_path: str, data: dict) -> None:
+    """
+    writes the given dictionary to a json file
+    :param file_path: the path of the json file
+    :param data: the data that will be dumped to json file.
+    """
+    with open(file_path, "w") as fw:
+        return json.dump(data, fw, indent=4, ensure_ascii=False)
+
+
+def convert_json_to_csv(file_path: str) -> str:
+    """
+    Function to read json file and send the data to the conversion function, then write the CSV in file
+    :param file_path: the path of the json file
+    :return: a string message denoting the completion of converting
+    """
+    json_data = read_json_file(file_path=f"{file_path}.json")
+    from json2csv import json_2_csv
+
+    csv = json_2_csv(json_data=json_data)
+    with open(f"./automated/{file_path}.csv", "w", encoding="utf-16") as csv_file:
+        csv_file.write(csv)
+    print("Completed converting json to csv.")
+
+
+def convert_csv_to_excel(csv_file_path: str) -> str:
+    """
+    Convert the generated csv from the previous function to Excel file
+    :param csv_file_path: the path of the csv file
+    :param excel_file_path: the path of the Excel file
+    :return: a message denoting the completion of conversion.
+    """
+    from csv2xlsx import csv_to_excel
+
+    csv_to_excel(csv_file_path=csv_file_path)
+    print("Completed converting csv to excel.")
+
+
+def convert_json_to_sql(file_path: str) -> str:
+    from json2sql import json_2_sql
+
+    json_file = read_json_file(file_path=f"{file_path}.json")
+    sql = json_2_sql(json_data=json_file, sqlite=True)
+    with open(f"./automated/{file_path}-sqlite.sql", "w", encoding="utf-16") as file:
+        file.write(sql)
+    print("Completed converting json to sqlite format.")
+    ############################################################################
+    # if you want to write the sql script to sqlite db, uncomment the next block
+    try:
+        import sqlite3
+
+        connection = sqlite3.connect(f"automated/{file_path}.db")
+        connection.executescript(sql)
+        print("wrote the sql to sqlite db successfully.")
+    except Exception as e:
+        print(f"error writing data: {e}")
+        connection.commit()
+    finally:
+        connection.close()
+    ############################################################################
+    sql = json_2_sql(json_data=json_file, sqlite=False)
+    with open(f"./automated/{file_path}-mysql.sql", "w", encoding="utf-16") as file:
+        file.write(sql)
+    print("Completed converting json to mysql format.")
+
+
+def convert_json_to_xml(file_path: str) -> str:
+    """
+    Convert json or dictionary to xml and write it to a file
+    :returns message donating the completion of conversion
+    """
+    from json2xml import xMl, json_2_xml
+
+    json_data = read_json_file(file_path=f"{file_path}.json")
+    root = xMl.Element("Palestine")
+    root = json_2_xml(root=root, json_data=json_data)
+    xMl.indent(root)
+    xMl.ElementTree(root).write(f"./automated/{file_path}.xml", encoding="utf-16")
+    print("Completed converting json to xml.")
+
+
+def convert_json_to_yaml(file_path: str) -> str:
+    """
+    Function to read json file and send the data to the conversion function, then write the YAML in file
+    :param file_path: the path of the json file
+    :return: a string message denoting the completion of converting
+    """
+    from json2yaml import json_2_yaml
+
+    json_data = read_json_file(file_path=f"{file_path}.json")
+    yaml = json_2_yaml(json_data=json_data)
+    with open(f"./automated/{file_path}.yml", "w", encoding="utf-16") as yaml_file:
+        yaml_file.write(yaml)
+    print("Completed converting json to yaml.")
+
+
+################################
+# Using Functions
+################################
+try:
+    os.mkdir("automated")
+except FileExistsError:
+    ...
+
+for json_file in glob.glob("*.json"):
+    file: str = json_file.split(".")[0]
+    convert_json_to_xml(file_path=file)
+    convert_json_to_yaml(file_path=file)
+    convert_json_to_csv(file_path=file)
+    convert_csv_to_excel(csv_file_path=f"./automated/{file}")
+    convert_json_to_sql(file_path=file)

util/csv2xlsx.py

@@ -0,0 +1,30 @@
+# Go to https://data.page/json/csv to convert (even to Excel format)
+
+# TODO
+# 1. make a code to convert yemen.json to yemen.xlsx
+# 2. utilize GitHub actions to automate this process after any changes in the yemen.json file
+import openpyxl as excel
+import csv
+
+
+def csv_to_excel(csv_file_path: str) -> None:
+    """
+    Function to convert csv content to Excel file
+    :param csv_file_path: the path of the csv file
+    :return: None
+    """
+
+    if not (isinstance(csv_file_path, str)):
+        raise ValueError("All the paths must be string.")
+
+    csv_data = []
+    with open(f"{csv_file_path}.csv", encoding="utf-16") as file_obj:
+        reader = csv.reader(file_obj)
+        for row in reader:
+            csv_data.append(row)
+    workbook = excel.Workbook()
+    sheet = workbook.active
+    sheet.title = "Yemen-Info"
+    for row in csv_data:
+        sheet.append(row)
+    workbook.save(f"{csv_file_path}.xlsx")

util/json2csv.py

@@ -0,0 +1,32 @@
+# Go to https://data.page/json/csv to convert
+
+# TODO
+# 1. make a code to convert yemen.json to yemen.csv
+# 2. utilize GitHub actions to automate this process after any changes in the yemen.json file
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+
+def json_2_csv(json_data: dict) -> str:
+    """
+    Function to send the json data object to the defined api_url to receive the csv version of the data
+    :param json_data: the dictionary to be converted to csv object
+    :returns: a csv text as string.
+    """
+
+    if not isinstance(json_data, dict):
+        raise ValueError("enter the json_data as dictionary.")
+
+    json_data = {
+        "json": json_data,
+    }
+
+    api_url = "https://data.page/api/getcsv"
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) "
+                      "Chrome/35.0.1916.47 Safari/537.36"
+    }
+
+    request = Request(api_url, urlencode(json_data).encode(), headers=headers)
+    csv = urlopen(request).read().decode()
+    return csv