added reference scraping

c_transpiler/convert.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from pathlib import Path
 from ir_transpiler import FunctionAssign
+from reference_scraping import reference_scraping
 
 
 from scanner import Scanner
@@ -9,17 +10,29 @@
 from parser import parser
 from ir_transpiler import ir_to_macro_request_rust, ir_to_py
 import inflection
+import sys
 
 
-def cpp_to_rust(assigns: list[FunctionAssign], rust_file: Path, api_name: str) -> None:
+@dataclass
+class API:
+    header: Path
+    api_name: str
+    file_name: str
+    description: str
+
+
+def cpp_to_rust(assigns: list[FunctionAssign], rust_file: Path, api: API) -> None:
 
     space = ' '*4
 
+    api_name = api.api_name
+
     trait_name = inflection.camelize(api_name)
     rust_assigns = [ir_to_macro_request_rust(
         assign, api_name) for assign in assigns]
     rust_string = ",\n".join(rust_assigns)
     content = 'use crate::RemoteApiClientInterface;\n'
+    content += f'''#[doc=r#"{api.description}"#]\n'''
     content += f'pub trait {trait_name} : RemoteApiClientInterface {{\n{space}requests!{{\n"{api_name}",\n{rust_string}\n}}\n}}'
 
     rust_file.write_text(content)
@@ -47,35 +60,63 @@ def cpp_to_py(assigns: list[FunctionAssign], py_file: Path, protocol_name: str)
     py_file.write_text(content)
 
 
-@dataclass
-class API:
-    header: Path
-    api_name: str
-    file_name: str
-
-
 def main():
     assets = Path('assets')
     sim_ik_h = assets / Path('sim_ik_api_header.h')
     sim_h = assets / Path('sim_api_header.h')
 
+    if len(sys.argv) != 2:
+        print('you shoud pass the coppeliasim dir as argument')
+        exit(1)
+    coppeliasim_dir = Path(sys.argv[1])
+    regular_api_dir = reference_scraping.get_regular_api_dir(coppeliasim_dir)
+    sim_ik_reference_file = reference_scraping.get_sim_ik_file(coppeliasim_dir)
+
+    sim_ik_description = reference_scraping.scraping_sim_ik_api_description(
+        coppeliasim_dir)
+    sim_description = reference_scraping.scraping_regular_api_description(
+        coppeliasim_dir)
     apis = [
-        API(sim_ik_h, 'simIK', 'sim_ik_api'),
-        API(sim_h, 'sim', 'sim_api'),
+        API(sim_ik_h, 'simIK', 'sim_ik_api', sim_ik_description),
+        API(sim_h, 'sim', 'sim_api', sim_description),
     ]
 
+    def sim_function_description(assign: FunctionAssign) -> str:
+        try:
+            descrip = reference_scraping.scraping_regular_api_function(
+                assign.function_name, regular_api_dir)
+        except Exception:
+            descrip = ''
+            print(f'Unable to find reference for sim {assign.function_name}')
+        return descrip
+
+    def sim_ik_function_description(assign: FunctionAssign) -> str:
+        try:
+            descrip = reference_scraping.scraping_sim_ik_api_function(
+                assign.function_name, sim_ik_reference_file)
+        except Exception:
+            descrip = ''
+            print(f'Unable to find reference for SimIK {assign.function_name}')
+        return descrip
+
+    descriptions = {apis[0].api_name: sim_ik_function_description,
+                    apis[1].api_name: sim_function_description}
+
     for api in apis:
         content = api.header.read_text()
         stream = StringStream(content)
         scanner = Scanner(stream)
 
         assigns = parser(scanner, stream)
 
+        for assign in assigns:
+            assign.description = descriptions[api.api_name](assign)
+
         rust_file = assets / Path(f'{api.file_name}.rs')
         python_file = assets / Path(f'{api.file_name}.py')
 
-        cpp_to_rust(assigns, rust_file, api.api_name)
-        cpp_to_py(assigns, python_file, api.api_name)
+        cpp_to_rust(assigns, rust_file, api)
+        # cpp_to_py(assigns, python_file, api.api_name)
 
 
 if __name__ == '__main__':

c_transpiler/ir_transpiler/ir.py

@@ -4,6 +4,7 @@
 from dataclasses import dataclass
 from cpp_token import TokenType
 
+
 @dataclass
 class TypeNode:
     cpp_type: TokenType
@@ -20,4 +21,5 @@ class Arg:
 class FunctionAssign:
     return_type: TypeNode
     function_name: str
-    function_args: list[Arg]
+    function_args: list[Arg]
+    description: str = ''

c_transpiler/ir_transpiler/ir_to_macro_request_rust.py

@@ -24,7 +24,8 @@ def ir_to_macro_request_rust(assign: FunctionAssign, api_name: str) -> str:
         option_args) != 0 else ""
 
     return_type_string = f'->{return_type}' if return_type else ""
-    return f'({rust_func_name},"{assign.function_name}"{required_args_string}{opt_string}{return_type_string})'
+    docstring = assign.description
+    return f'(r#"{docstring}"#,{rust_func_name},"{assign.function_name}"{required_args_string}{opt_string}{return_type_string})'
 
 
 def arg_to_rust(arg: Arg) -> str:

c_transpiler/reference_scraping/__init__.py

@@ -0,0 +1 @@
+

c_transpiler/reference_scraping/reference_scraping.py

@@ -0,0 +1,132 @@
+from pathlib import Path
+import inflection
+from bs4 import BeautifulSoup
+
+
+def scraping_regular_api(function_names: list[str], coppeliasim_dir: Path) -> dict[str, str]:
+    """Expecting function_names like: [ 'loadScene','setJointMaxForce'], and the simulator dir"""
+    regular_api_dir = get_regular_api_dir(coppeliasim_dir)
+
+    function_descriptions = {}
+
+    for function_name in function_names:
+        description = scraping_regular_api_function(
+            function_name,
+            regular_api_dir)
+        function_descriptions[function_name] = description
+
+    return function_descriptions
+
+
+def scraping_regular_api_description(coppeliasim_dir: Path) -> str:
+    en = get_regular_api_dir(coppeliasim_dir).parent
+    index = en / 'apiFunctions.htm'
+    soup = BeautifulSoup(index.read_text(), 'html.parser')
+    description = _get_text_for_warning_box(soup)
+    return description
+
+
+def scraping_sim_ik_api_description(coppeliasim_dir: Path) -> str:
+    sim_ik_file = get_sim_ik_file(coppeliasim_dir)
+    soup = BeautifulSoup(sim_ik_file.read_text(), 'html.parser')
+    description = _get_text_for_warning_box(soup)
+    return description
+
+
+def _get_text_for_warning_box(soup: BeautifulSoup) -> str:
+
+    p = soup.find("p", {"class": "warningBox"})
+    if p is None:
+        raise Exception('Unable to find Regular API description')
+
+    return p.text
+
+
+def scraping_sim_ik_api(function_names: list[str], coppeliasim_dir: Path) -> dict[str, str]:
+
+    function_descriptions = {}
+
+    sim_ik_file = get_sim_ik_file(coppeliasim_dir)
+
+    for name in function_names:
+        description = scraping_sim_ik_api_function(name, sim_ik_file)
+        function_descriptions[name] = description
+
+    return function_descriptions
+
+
+def get_regular_api_dir(coppeliasim_dir: Path) -> Path:
+    return coppeliasim_dir/'helpFiles'/'en'/'regularApi'
+
+
+def get_sim_ik_file(coppeliasim_dir: Path) -> Path:
+    return coppeliasim_dir/'helpFiles'/'en'/'simIK.htm'
+
+
+def scraping_regular_api_function(function_name: str, regular_api_dir: Path) -> str:
+
+    name = inflection.camelize(function_name)
+    reference_function_name = f'sim{name}'
+    reference_file = regular_api_dir / f'{reference_function_name}.htm'
+    if not reference_file.exists():
+        raise Exception(
+            f'Unable to find {function_name} and your file: {reference_file}')
+
+    html = reference_file.read_text()
+    description = _scraping_sim_api_html(html)
+
+    return description
+
+
+def scraping_sim_ik_api_function(function_name: str, sim_ik_file: Path) -> str:
+    name = f'simIK.{function_name}'  # id="simIK.addIkElement"
+    soup = BeautifulSoup(sim_ik_file.read_text(), 'html.parser')
+    a_tag = soup.find("a", {"id": name})
+
+    if a_tag is None:
+        raise Exception(
+            f'Unable to find {function_name} in SimIK')
+
+    # getting the next table of p
+    '''
+        <p> <a id="SimIK.addElement"> </p>
+        <table>
+                <tr>
+                    <td class="apiTableLeftDescr">Description</td>
+                    <td class="apiTableRightDescr">Adds a new IK element to an IK group.</td>
+                </tr>
+                ...
+        </table>
+    '''
+    p_tag = a_tag.parent
+
+    if p_tag is None:
+        raise Exception(
+            f'Unable to find p tag of{function_name} in SimIK')
+    table_tag = p_tag.next_sibling
+
+    if table_tag is None:
+        raise Exception(
+            f'Unable to find table tag of{function_name} in SimIK')
+
+    td_tag = table_tag.find_next("td", {"class": "apiTableRightDescr"})
+
+    if td_tag is None:
+        raise Exception(
+            f'Unable to find description tag of{function_name} in SimIK')
+
+    description = td_tag.text
+
+    return description
+
+
+def _scraping_sim_api_html(html: str) -> str:
+    """returning the function description"""
+    soup = BeautifulSoup(html, 'html.parser')
+
+    description_td = soup.find("td", {"class": "apiTableRightDescr"})
+
+    if description_td is None:
+        raise Exception('Unable to find description by class')
+
+    return description_td.text

c_transpiler/requirements.txt

@@ -1,4 +1,6 @@
 autopep8==1.7.0
+beautifulsoup4==4.12.2
 inflection==0.5.1
 pycodestyle==2.9.1
+soupsieve==2.5
 toml==0.10.2