add browser automation for copilot

seii-saintway · Jan 9, 2025 · 843f8e3 · 843f8e3
1 parent d4d80f3
commit 843f8e3
Show file tree

Hide file tree

Showing 11 changed files with 478 additions and 119 deletions.
diff --git a/docs/browser.html b/docs/browser.html
diff --git a/docs/tools_duckduckgo.html b/docs/tools_duckduckgo.html
@@ -110,12 +110,12 @@ <h2 id="DuckDuckGoSearchAPIWrapper" class="doc_header"><code>class</code> <code>
             <span class="n">max_results</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_results</span><span class="p">,</span>
         <span class="p">)</span>
         <span class="k">if</span> <span class="n">results</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="n">results</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-            <span class="k">return</span> <span class="sa">f</span><span class="s1">'搜索「</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s1">」没有发现好的｛DuckDuckGo 搜索结果：</span><span class="si">{</span><span class="n">results</span><span class="si">}</span><span class="s1">｝'</span>
+            <span class="k">return</span> <span class="sa">f</span><span class="s1">'No suitable DuckDuckGo search results found for "</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s1">": </span><span class="se">{{</span><span class="si">{</span><span class="n">results</span><span class="si">}</span><span class="se">}}</span><span class="s1">'</span>
         <span class="n">snippets</span> <span class="o">=</span> <span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">result</span><span class="p">[</span><span class="s1">'body'</span><span class="p">]</span> <span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">results</span><span class="p">])</span>
         <span class="k">return</span> <span class="p">(</span>
-            <span class="sa">f</span><span class="s1">'用 DuckDuckGo 搜索「</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s1">」的结果：「</span><span class="se">\n</span><span class="s1">'</span>
+            <span class="sa">f</span><span class="s1">'Results for the DuckDuckGo search "</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s1">": </span><span class="se">{{\n</span><span class="s1">'</span>
             <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">snippets</span><span class="si">}</span><span class="se">\n</span><span class="s1">'</span>
-            <span class="sa">f</span><span class="s1">'」'</span>
+            <span class="sa">f</span><span class="s1">'</span><span class="se">}}</span><span class="s1">'</span>
         <span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">results</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">num_results</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Dict</span><span class="p">]:</span>

diff --git a/docs/tools_pyknp.html b/docs/tools_pyknp.html
@@ -50,7 +50,7 @@
 
 
 <div class="output_markdown rendered_html output_subarea ">
-<h4 id="is_halfwidth" class="doc_header"><code>is_halfwidth</code><a href="https://github.com/seii-saintway/ipymock/tree/main/ipymock/nlp.py#L11" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>is_halfwidth</code>(<strong><code>text</code></strong>)</p>
+<h4 id="is_halfwidth" class="doc_header"><code>is_halfwidth</code><a href="https://github.com/seii-saintway/ipymock/tree/main/ipymock/nlp.py#L9" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>is_halfwidth</code>(<strong><code>text</code></strong>)</p>
 </blockquote>
 <p>Determine whether the text consists entirely of halfwidth characters.
 :param text: Input text string
@@ -78,7 +78,7 @@ <h4 id="is_halfwidth" class="doc_header"><code>is_halfwidth</code><a href="https
 
 
 <div class="output_markdown rendered_html output_subarea ">
-<h4 id="halfwidth_to_fullwidth" class="doc_header"><code>halfwidth_to_fullwidth</code><a href="https://github.com/seii-saintway/ipymock/tree/main/ipymock/nlp.py#L19" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>halfwidth_to_fullwidth</code>(<strong><code>text</code></strong>)</p>
+<h4 id="halfwidth_to_fullwidth" class="doc_header"><code>halfwidth_to_fullwidth</code><a href="https://github.com/seii-saintway/ipymock/tree/main/ipymock/nlp.py#L17" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>halfwidth_to_fullwidth</code>(<strong><code>text</code></strong>)</p>
 </blockquote>
 
 </div>
@@ -103,7 +103,48 @@ <h4 id="halfwidth_to_fullwidth" class="doc_header"><code>halfwidth_to_fullwidth<
 
 
 <div class="output_markdown rendered_html output_subarea ">
-<h4 id="annotate" class="doc_header"><code>annotate</code><a href="https://github.com/seii-saintway/ipymock/tree/main/ipymock/nlp.py#L32" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>annotate</code>(<strong><code>text</code></strong>)</p>
+<h4 id="annotate" class="doc_header"><code>annotate</code><a href="https://github.com/seii-saintway/ipymock/tree/main/ipymock/nlp.py#L30" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>annotate</code>(<strong><code>text</code></strong>)</p>
+</blockquote>
+
+</div>
+
+</div>
+
+</div>
+</div>
+
+</div>
+    {% endraw %}
+
+    {% raw %}
+
+<div class="cell border-box-sizing code_cell rendered">
+
+</div>
+    {% endraw %}
+
+<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
+<div class="text_cell_render border-box-sizing rendered_html">
+<div class="highlight"><pre><span></span>pip install beautifulsoup4
+pip install lxml
+</pre></div>
+
+</div>
+</div>
+</div>
+    {% raw %}
+
+<div class="cell border-box-sizing code_cell rendered">
+
+<div class="output_wrapper">
+<div class="output">
+
+<div class="output_area">
+
+
+
+<div class="output_markdown rendered_html output_subarea ">
+<h4 id="annotate_html" class="doc_header"><code>annotate_html</code><a href="https://github.com/seii-saintway/ipymock/tree/main/ipymock/nlp.py#L56" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>annotate_html</code>(<strong><code>content</code></strong>, <strong><code>interactive</code></strong>=<em><code>False</code></em>)</p>
 </blockquote>
 
 </div>

diff --git a/ipymock/_nbdev.py b/ipymock/_nbdev.py
@@ -9,8 +9,8 @@
          "device_pixel_ratio": "2_automation.ipynb",
          "init": "2_browser.ipynb",
          "quit": "2_automation.ipynb",
-         "logger": "2_automation.ipynb",
-         "handler": "2_automation.ipynb",
+         "logger": "2_browser.ipynb",
+         "handler": "2_browser.ipynb",
          "ok": "2_automation.ipynb",
          "last": "2_automation.ipynb",
          "new": "2_automation.ipynb",
@@ -88,6 +88,7 @@
          "chatgpt_red_500": "2_browser.ipynb",
          "chatgpt_big_response": "2_browser.ipynb",
          "chatgpt_small_response": "2_browser.ipynb",
+         "input_prompt": "2_browser.ipynb",
          "request": "2_browser.ipynb",
          "get_last_response": "2_browser.ipynb",
          "get_response": "2_browser.ipynb",
@@ -132,7 +133,8 @@
          "DuckDuckGoSearchAPIWrapper": "4_tools_duckduckgo.ipynb",
          "is_halfwidth": "4_tools_pyknp.ipynb",
          "halfwidth_to_fullwidth": "4_tools_pyknp.ipynb",
-         "annotate": "4_tools_pyknp.ipynb"}
+         "annotate": "4_tools_pyknp.ipynb",
+         "annotate_html": "4_tools_pyknp.ipynb"}
 
 modules = ["__init__.py",
            "automation.py",

diff --git a/ipymock/browser.py b/ipymock/browser.py
@@ -2,9 +2,9 @@
 
 __all__ = ['common', 'get_conversations', 'get_conversation', 'handle_conversation_detail', 'start_conversation',
            'generate_title', 'rename_title', 'delete_conversation', 'recover_conversation', 'clear_conversations',
-           'init', 'login', 'open_chat', 'remove_portal', 'request', 'get_last_response', 'get_response', 'ask',
-           'get_screenshot', 'attrdict', 'attributize', 'retry_on_status_code', 'content', 'new_id', 'delta',
-           'chat_delta', 'mock_create', 'mock_chat_create', 'mock_openai']
+           'init', 'login', 'open_chat', 'remove_portal', 'input_prompt', 'request', 'get_last_response',
+           'get_response', 'ask', 'get_screenshot', 'attrdict', 'attributize', 'retry_on_status_code', 'content',
+           'new_id', 'delta', 'chat_delta', 'mock_create', 'mock_chat_create', 'mock_openai']
 
 # Internal Cell
 from queue import Queue
@@ -43,6 +43,19 @@ class Common:
     common.access_token = common.config.get('access_token', common.access_token)
     common.conversation_id = common.config.get('conversation_id', common.conversation_id)
 
+# Internal Cell
+import logging
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+handler = logging.StreamHandler()
+handler.setFormatter(logging.Formatter(
+    fmt = '[%(asctime)s][%(levelname)s]<%(name)s> %(message)s',
+    datefmt = '%H:%M:%S'
+))
+logger.addHandler(handler)
+
 # Cell
 def get_conversations():
     response = requests.get(f'{common.chat_gpt_base_url}/conversations?offset=0&limit=100', headers = {'Authorization': f'Bearer {common.access_token}'})
@@ -289,7 +302,22 @@ def init(chrome_args = set()):
 
 # Cell
 def login():
-    new('https://chatgpt.com/auth/login')
+    if 'github' in common.chat_gpt_base_url:
+        new('https://github.com/login')
+        wait(5.0)
+        input(common.config['email'], 'Username or email address', yoffset = 35)
+        wait(5.0)
+        input(common.config['password'], 'Password', yoffset = 35)
+        wait(5.0)
+        click('Sign in')
+        wait(5.0)
+        click('Use passkey')
+        wait(stability_duration = 5.0)
+        common.driver.maximize_window()
+        wait(1.0)
+        return
+
+    new(f'{common.chat_gpt_base_url}/auth/login')
 
     # WebDriverWait(common.driver, 5).until(
     #     expected_conditions.presence_of_element_located((By.XPATH, '//*[text()="Log in"]'))
@@ -390,9 +418,9 @@ def open_chat(conversation_id = ''):
     from .automation import driver
     common.driver = driver
     if conversation_id == '':
-        common.driver.get('https://chatgpt.com/')
+        common.driver.get(f'{common.chat_gpt_base_url}/')
     else:
-        common.driver.get(f'https://chatgpt.com/c/{conversation_id}')
+        common.driver.get(f'{common.chat_gpt_base_url}/c/{conversation_id}')
         if common.conversation_id != conversation_id:
             common.conversation_id = conversation_id
             common.parent_message_id = ''
@@ -402,6 +430,8 @@ def open_chat(conversation_id = ''):
     # )
     wait(5.0)
 
+    if 'copilot' in common.chat_gpt_base_url:
+        return
     remove_portal()
 
 def remove_portal():
@@ -446,7 +476,36 @@ def remove_portal():
 # from ipymock.automation import exists, touch
 
 # Cell
+def input_prompt(prompt):
+    # textbox.send_keys(prompt.strip())
+    # common.driver.execute_script('''
+    # var element = arguments[0], txt = arguments[1];
+    # element.value += txt;
+    # element.dispatchEvent(new Event("change"));
+    # ''',
+    #     textbox,
+    #     prompt.strip(),
+    # )
+    for line in prompt.strip().split('\n'):
+        fill(line)
+        ActionChains(common.driver).key_down(Keys.SHIFT).send_keys(Keys.ENTER).key_up(Keys.SHIFT).perform()
+
+    # WebDriverWait(common.driver, 3).until_not(
+    #     expected_conditions.presence_of_element_located(chatgpt_disabled_button)
+    # )
+    wait(stability_duration = 3.0)
+
+    # textbox.send_keys('\n')
+    # textbox.send_keys(Keys.ENTER)
+
 def request(prompt: str) -> None:
+    if 'copilot' in common.chat_gpt_base_url:
+        click('Ask Copilot')
+        input_prompt(prompt)
+        fill(Keys.ENTER)
+        wait(1.0)
+        return
+
     # try:
     #     textbox = WebDriverWait(common.driver, 5).until(
     #         expected_conditions.element_to_be_clickable(chatgpt_textbox)
@@ -472,26 +531,7 @@ def request(prompt: str) -> None:
     # touch(textbox)
     click('Message ChatGPT')
 
-    # textbox.send_keys(prompt.strip())
-    # common.driver.execute_script('''
-    # var element = arguments[0], txt = arguments[1];
-    # element.value += txt;
-    # element.dispatchEvent(new Event("change"));
-    # ''',
-    #     textbox,
-    #     prompt.strip(),
-    # )
-    for line in prompt.strip().split('\n'):
-        fill(line)
-        ActionChains(common.driver).key_down(Keys.SHIFT).send_keys(Keys.ENTER).key_up(Keys.SHIFT).perform()
-
-    # WebDriverWait(common.driver, 3).until_not(
-    #     expected_conditions.presence_of_element_located(chatgpt_disabled_button)
-    # )
-    wait(stability_duration = 3.0)
-
-    # textbox.send_keys('\n')
-    # textbox.send_keys(Keys.ENTER)
+    input_prompt(prompt)
 
     # click('ChatGPT can make mistakes. Check important info.')
     # send_button = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../assets/send-button.png'))
@@ -509,15 +549,59 @@ def request(prompt: str) -> None:
     #     pass
 
 def get_last_response():
+    if 'copilot' in common.chat_gpt_base_url:
+        chatgpt_response = (By.XPATH, '//div[contains(@class, "markdown-body")]')
+        chatgpt_big_response = (By.XPATH, '//div[starts-with(@class, "js-snippet-clipboard-copy-unpositioned")]//div[p or pre]')
+        chatgpt_small_response = (By.XPATH, './/code[div]')
     for xpath in chatgpt_response, chatgpt_big_response:
-        responses = common.driver.find_elements(*xpath)
+        while True:
+            responses = common.driver.find_elements(*xpath)
+            elements = []
+            if responses != []:
+                try:
+                    elements = responses[-1].find_elements(*chatgpt_small_response)
+                except StaleElementReferenceException:
+                    continue
+            break
+        if len(elements) == 1:
+            return elements[0]
         if responses != []:
-            elements = responses[-1].find_elements(*chatgpt_small_response)
-            if len(elements) == 1:
-                return elements[0]
             return responses[-1]
 
 def get_response() -> Generator[str, None, None]:
+    if 'copilot' in common.chat_gpt_base_url:
+        from .automation import get_html_hash
+        # Get the initial hash value
+        previous_hash, previous_time = get_html_hash()
+        response = get_last_response()
+
+        # Wait until the HTML does not change
+        start_time = time.time()
+        while True:
+            time.sleep(0.1)
+
+            # Get the current hash value
+            current_hash, current_time = get_html_hash()
+            response = get_last_response()
+
+            # Check if the hash value has stabilized
+            if current_hash == previous_hash:
+                if current_time - previous_time >= 5.0:
+                    logger.info('HTML content has stabilized.')
+                    break
+            else:
+                # Update hash and time if the content changes
+                previous_hash, previous_time = current_hash, current_time
+            yield markdownize(response.get_attribute('innerHTML'))
+
+            # Check for timeout
+            if current_time - start_time >= float('inf'):
+                logger.info('Wait for HTML stabilization timed out.')
+                break
+        response = get_last_response()
+        yield markdownize(response.get_attribute('innerHTML'))
+        return
+
     try:
         result_streaming = WebDriverWait(common.driver, 5).until(
             expected_conditions.presence_of_element_located(chatgpt_streaming)

diff --git a/ipymock/nlp.py b/ipymock/nlp.py
@@ -1,6 +1,6 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: nbs/4_tools_pyknp.ipynb (unless otherwise specified).
 
-__all__ = ['is_halfwidth', 'halfwidth_to_fullwidth', 'annotate']
+__all__ = ['is_halfwidth', 'halfwidth_to_fullwidth', 'annotate', 'annotate_html']
 
 # Internal Cell
 from pyknp import Juman
@@ -45,4 +45,40 @@ def annotate(text):
                 yield mrph.midasi
                 continue
             yield f'<ruby>{mrph.midasi}<rt>{mrph.yomi}</rt></ruby>'
-        yield '\n'
+        yield '\n'
+
+# Internal Cell
+import bs4
+from .nlp import annotate
+from IPython.display import display, HTML
+
+# Cell
+def annotate_html(content, interactive = False):
+    # Parse the HTML content with BeautifulSoup
+    soup = bs4.BeautifulSoup(content, 'lxml-xml')
+
+    # Iterate through the div elements and process only the leaf nodes
+    for div in soup.find_all('div'):  # Find all 'div' elements in the soup
+        # Check if the div is a leaf node (contains only text)
+        if div.find_all(True):  # Has child tags, so it's a leaf node
+            continue
+        try:
+            answser = ''
+            line = ''
+            for word in annotate(div.get_text()):
+                answser += word
+                if word == '\n':
+                    if interactive:
+                        display(HTML(line))
+                    line = ''
+                else:
+                    line += word
+            # Replace the content of the div with parsed HTML
+            new_content = bs4.BeautifulSoup(answser, 'html.parser')
+            div.clear()  # Clear the original content
+            div.append(new_content)  # Append the new parsed content
+        except Exception as e:
+            print(f'Error processing content: {e}')
+            continue
+
+    return str(soup)  # Convert the soup back to string
diff --git a/ipymock/reader.py b/ipymock/reader.py
@@ -50,12 +50,12 @@ def run(self, query: str) -> str:
             max_results=self.max_results,
         )
         if results is None or len(results) == 0:
-            return f'搜索「{query}」没有发现好的｛DuckDuckGo 搜索结果：{results}｝'
+            return f'No suitable DuckDuckGo search results found for "{query}": {{{results}}}'
         snippets = '\n'.join([result['body'] for result in results])
         return (
-            f'用 DuckDuckGo 搜索「{query}」的结果：「\n'
+            f'Results for the DuckDuckGo search "{query}": {{\n'
             f'{snippets}\n'
-            f'」'
+            f'}}'
         )
 
     def results(self, query: str, num_results: int) -> List[Dict]: