Merge pull request #132 from sam80180/main

fix: support W3C touch actions (for WDA version >= 7.0.0)

pom.xml

@@ -101,6 +101,22 @@
  <version>4.6.1</version>
  <scope>test</scope>
  </dependency>
+ <dependency>
+ <groupId>com.github.zafarkhaja</groupId>
+ <artifactId>java-semver</artifactId>
+ <version>0.10.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.seleniumhq.selenium</groupId>
+ <artifactId>selenium-api</artifactId>
+ <version>4.22.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>4.4.16</version>
+ </dependency>
+
 
  <!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
  <dependency>

src/main/java/org/cloud/sonic/driver/common/models/WDAStatus.java

@@ -0,0 +1,24 @@
+package org.cloud.sonic.driver.common.models;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.ToString;
+
+@Getter
+@ToString
+@AllArgsConstructor
+public class WDAStatus {
+ private WDABuild build;
+
+ @Getter
+ @ToString
+ @AllArgsConstructor
+ public static class WDABuild {
+ private String version;
+ } // end class
+} // end class
+
+/*
+References:
+https://github.com/SonicCloudOrg/sonic-driver-core/blob/faa0948e11e02be04db3ec9754fb66d613cf8bfa/src/main/java/org/cloud/sonic/driver/ios/service/impl/WdaClientImpl.java#L128
+*/

src/main/java/org/cloud/sonic/driver/common/tool/SemanticVersionTools.java

@@ -0,0 +1,25 @@
+package org.cloud.sonic.driver.common.tool;
+
+import com.github.zafarkhaja.semver.Version;
+
+public class SemanticVersionTools {
+ public static Version parseSemVer(final String s) {
+ return Version.parse(s);
+ } // end parseSemVer()
+
+ public static Version getVersionCore(final Version v) {
+ return Version.of(v.majorVersion(), v.minorVersion(), v.patchVersion());
+ } // end getVersionCore()
+
+ public static String pad4SemVer(final String s) {
+ if (Version.isValid(s)) { return s; } // end if
+ final String p[] = s.split("\\.");
+ final String vv[] = new String[] {"0", "0", "0"};
+ for (int i=0; i<3; i++) {
+ try {
+ vv[i] = String.format("%d", Integer.parseInt(p[i], 10));
+ } catch (Exception e) {} // end try
+ } // end for
+ return String.join(".", vv);
+ } // end pad4SemVer()
+} // end class

src/main/java/org/cloud/sonic/driver/common/tool/StringTool.java

@@ -0,0 +1,22 @@
+package org.cloud.sonic.driver.common.tool;
+
+import java.util.UUID;
+
+public class StringTool {
+ private static String[] CHARS = new String[] {
+ "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
+ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
+ "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"
+ };
+
+ public static String generateShortUuid() { // http://www.java2s.com/example/java-utility-method/uuid-create/generateshortuuid-db743.html
+ StringBuilder shortBuilder = new StringBuilder();
+ String uuid = UUID.randomUUID().toString().replace("-", "");
+ for (int i=0; i<8; i++) {
+ String str = uuid.substring(i*4, i*4+4);
+ int x = Integer.parseInt(str, 16);
+ shortBuilder.append(CHARS[x%0x3E]);
+ } // end for
+ return shortBuilder.toString();
+ } // end generateShortUuid()
+} // end class

src/main/java/org/cloud/sonic/driver/ios/IOSDriver.java

@@ -16,10 +16,20 @@
  */
 package org.cloud.sonic.driver.ios;
 
+import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
+import com.github.zafarkhaja.semver.Version;
+
+import cn.hutool.http.HttpUtil;
+import cn.hutool.http.Method;
+
 import org.cloud.sonic.driver.common.enums.PasteboardType;
+import org.cloud.sonic.driver.common.models.BaseResp;
+import org.cloud.sonic.driver.common.models.ErrorMsg;
+import org.cloud.sonic.driver.common.models.WDAStatus;
 import org.cloud.sonic.driver.common.models.WindowSize;
 import org.cloud.sonic.driver.common.tool.RespHandler;
+import org.cloud.sonic.driver.common.tool.SemanticVersionTools;
 import org.cloud.sonic.driver.common.tool.SonicRespException;
 import org.cloud.sonic.driver.ios.enums.*;
 import org.cloud.sonic.driver.ios.models.TouchActions;
@@ -36,6 +46,7 @@
  */
 public class IOSDriver {
  private WdaClient wdaClient;
+ private Version wdaVersion;
 
  /**
  * Init ios driver.
@@ -82,6 +93,7 @@ public IOSDriver(String url, int timeOut, JSONObject cap) throws SonicRespExcept
  wdaClient.setRemoteUrl(url);
  wdaClient.setGlobalTimeOut(timeOut);
  wdaClient.newSession(cap);
+ this.wdaVersion = this.getWDAVersion();
  }
 
  /**
@@ -230,7 +242,11 @@ public void swipe(double fromX, double fromY, double toX, double toY, double dur
  * @throws SonicRespException
  */
  public void performTouchAction(TouchActions touchActions) throws SonicRespException {
- wdaClient.performTouchAction(touchActions);
+ if (this.wdaVersion.satisfies("<7.0.0")) { 
+ this.wdaClient.performTouchAction(touchActions);
+ } else {
+ this.wdaClient.performW3CTouchAction(touchActions);
+ } // end if
  }
 
  /**
@@ -699,4 +715,16 @@ public void rotate(Orientation orientation) throws SonicRespException {
  public Orientation getRotate() throws SonicRespException {
  return wdaClient.getRotate();
  }
+
+ public Version getWDAVersion() throws SonicRespException {
+ this.wdaClient.checkSessionId();
+ final String url = this.wdaClient.getRemoteUrl()+"/status";
+ final BaseResp<?> b = this.wdaClient.getRespHandler().getResp(HttpUtil.createRequest(Method.GET, url));
+ if (b==null) { throw new SonicRespException("null response body"); } // end if
+ final ErrorMsg err = b.getErr();
+ if (err!=null) { throw new SonicRespException(err.getMessage()); } // end if
+ final String result = b.getValue().toString();
+ final WDAStatus sessionStatus = JSON.parseObject(result, WDAStatus.class);
+ return SemanticVersionTools.parseSemVer(SemanticVersionTools.pad4SemVer(sessionStatus.getBuild().getVersion()));
+ } // end getWDAVersion()
 }

src/main/java/org/cloud/sonic/driver/ios/models/W3CActions.java

@@ -0,0 +1,49 @@
+package org.cloud.sonic.driver.ios.models;
+
+import java.time.Duration;
+import org.cloud.sonic.driver.common.tool.StringTool;
+import org.cloud.sonic.driver.ios.enums.ActionType;
+import org.cloud.sonic.driver.ios.models.TouchActions.TouchAction;
+import org.cloud.sonic.driver.ios.models.TouchActions.TouchAction.Options;
+import org.openqa.selenium.interactions.Pause;
+import org.openqa.selenium.interactions.PointerInput;
+import org.openqa.selenium.interactions.Sequence;
+import org.openqa.selenium.interactions.PointerInput.Kind;
+import org.openqa.selenium.interactions.PointerInput.MouseButton;
+import org.openqa.selenium.interactions.PointerInput.Origin;
+
+public class W3CActions {
+ public static Sequence convert(final TouchActions touchActions) {
+ final PointerInput FINGER = new PointerInput(Kind.TOUCH, "finger-"+StringTool.generateShortUuid());
+ final Sequence seq = new Sequence(FINGER, 0 /* https://github.com/appium/appium/issues/11273#issuecomment-416636734 */);
+ for (TouchAction action: touchActions.getActions()) {
+ final String strAction = action.getAction();
+ final Options actionData = action.getOptions();
+ if (strAction.equals(ActionType.MOVE.getType())) {
+ final Integer optMs = actionData.getMs();
+ final long ms = (optMs==null ? 0 : Math.max(optMs, 0));
+ seq.addAction(FINGER.createPointerMove(Duration.ofMillis(ms), Origin.viewport(), actionData.getX(), actionData.getY()));
+ } else if (strAction.equals(ActionType.PRESS.getType())) {
+ final Integer optMs = actionData.getMs();
+ final long ms = (optMs==null ? 0 : Math.max(optMs, 0));
+ seq.addAction(FINGER.createPointerMove(Duration.ofMillis(ms), Origin.viewport(), actionData.getX(), actionData.getY()));
+ seq.addAction(FINGER.createPointerDown(MouseButton.LEFT.asArg()));
+ } else if (strAction.equals(ActionType.RELEASE.getType())) {
+ seq.addAction(FINGER.createPointerUp(MouseButton.LEFT.asArg()));
+ } else if (strAction.equals(ActionType.WAIT.getType())) {
+ final Integer optMs = actionData.getMs();
+ final long ms = (optMs==null ? 0 : Math.max(optMs, 0));
+ if (ms<=0) { continue; } // end if
+ seq.addAction(new Pause(FINGER, Duration.ofMillis(ms)));
+ } // end if
+ } // end for
+ return seq;
+ } // end convert()
+} // end class
+
+/*
+References:
+https://stackoverflow.com/a/71038411/12857692
+https://github.com/appium/appium-espresso-driver/issues/244
+https://github.com/appium/appium/issues/11273
+*/

src/main/java/org/cloud/sonic/driver/ios/service/WdaClient.java

@@ -71,6 +71,8 @@ public interface WdaClient {
 
  //perform handler.
  void performTouchAction(TouchActions touchActions) throws SonicRespException;
+
+ void performW3CTouchAction(final TouchActions touchActions) throws SonicRespException;
 
  //button handler.
  void pressButton(String buttonName) throws SonicRespException;

src/main/java/org/cloud/sonic/driver/ios/service/impl/WdaClientImpl.java

@@ -20,17 +20,20 @@
 import cn.hutool.http.Method;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
+import org.apache.http.entity.ContentType;
 import org.cloud.sonic.driver.common.models.BaseResp;
+import org.cloud.sonic.driver.common.models.ErrorMsg;
 import org.cloud.sonic.driver.common.models.SessionInfo;
 import org.cloud.sonic.driver.common.models.WindowSize;
 import org.cloud.sonic.driver.common.tool.Logger;
 import org.cloud.sonic.driver.common.tool.RespHandler;
 import org.cloud.sonic.driver.common.tool.SonicRespException;
 import org.cloud.sonic.driver.ios.enums.Orientation;
 import org.cloud.sonic.driver.ios.models.TouchActions;
+import org.cloud.sonic.driver.ios.models.W3CActions;
 import org.cloud.sonic.driver.ios.service.IOSElement;
 import org.cloud.sonic.driver.ios.service.WdaClient;
-import org.jsoup.select.CombiningEvaluator;
+import org.openqa.selenium.interactions.Sequence;
 
 import java.nio.charset.StandardCharsets;
 import java.util.*;
@@ -214,6 +217,18 @@ public void performTouchAction(TouchActions touchActions) throws SonicRespExcept
  throw new SonicRespException(b.getErr().getMessage());
  }
  }
+
+ @SuppressWarnings("serial")
+ @Override
+ public void performW3CTouchAction(final TouchActions touchActions) throws SonicRespException {
+ final Sequence seq = W3CActions.convert(touchActions);
+ final int timeoutUpdateSettings = 1500;
+ final String strAppiumSettingsURL = "%s/session/%s/actions".formatted(this.remoteUrl, this.sessionId);
+ final JSONObject payload = new JSONObject();
+ payload.put("actions", new LinkedList<Map<String, Object>>() {{ add(seq.toJson()); }});
+ final byte[] rawPayload = payload.toJSONString().getBytes(StandardCharsets.UTF_8);
+ this.sendAppiumRequest(Method.POST, strAppiumSettingsURL, ContentType.APPLICATION_JSON.toString(), rawPayload, timeoutUpdateSettings);
+ } // end performW3CTouchAction()
 
  @Override
  public void pressButton(String buttonName) throws SonicRespException {
@@ -607,4 +622,17 @@ public Orientation getRotate() throws SonicRespException {
  throw new SonicRespException(b.getErr().getMessage());
  }
  }
+
+ private void sendAppiumRequest(final Method method, final String url, final String contentType, final byte[] bodyBytes, final int timeout) throws SonicRespException {
+ this.checkSessionId();
+ final BaseResp<?> b = this.getRespHandler().getResp(HttpUtil.createRequest(method, url).body(bodyBytes).contentType(contentType), timeout);
+ if (b!=null) {
+ final ErrorMsg err = b.getErr();
+ if (err!=null) {
+ throw new SonicRespException(err.getMessage());
+ } // end if
+ } else {
+ throw new SonicRespException("null response body");
+ } // end if
+ } // end sendAppiumRequest()
 }