Espresso fps dom

visual-espresso/README.md

@@ -10,11 +10,13 @@ View installation and usage instructions on the [Sauce Docs website](https://doc
 
 Sauce Visual Java SDK uses [Gradle](https://gradle.org/).
 
-`gradlew` binary, that is included in the source, can be used as a replacement if you don't have Maven.
+`gradlew` binary, that is included in the source, can be used as a replacement if you don't have Gradle.
 
 You'll also need [Android command line tools](https://developer.android.com/tools/).
 
-It can be setup either using [Android Studio](https://developer.android.com/studio) or using [homebrew](https://formulae.brew.sh/cask/android-commandlinetools).
+It can be installed either using [Android Studio](https://developer.android.com/studio) or using [homebrew](https://formulae.brew.sh/cask/android-commandlinetools).
+
+Finally, the library can be built using the following command:
 
 ```sh
 ./gradlew build
@@ -24,11 +26,13 @@ It can be setup either using [Android Studio](https://developer.android.com/stud
 
 To run the smoke test you'll need a running Android Emulator.
 
-You can either start an emulator from Android Studio or using [command line](https://developer.android.com/studio/run/emulator-commandline).
+You can start an emulator either from Android Studio or using [command line](https://developer.android.com/studio/run/emulator-commandline).
 
-Then you can run the smoke test using the following command. Make sure that SAUCE_USERNAME and SAUCE_ACCESS_KEY environment variables
+Make sure that SAUCE_USERNAME and SAUCE_ACCESS_KEY environment variables
 are in place before running the test.
 
+The smoke test can be run using the following command: 
+
 ```sh
 ./gradlew connectedAndroidTest
 ```

visual-espresso/gradle/libs.versions.toml

@@ -18,11 +18,11 @@ espresso-core = { group = "androidx.test.espresso", name = "espresso-core", vers
 appcompat = { group = "androidx.appcompat", name = "appcompat", version.ref = "appcompat" }
 material = { group = "com.google.android.material", name = "material", version.ref = "material" }
 uiautomator = { group = "androidx.test.uiautomator", name = "uiautomator", version.ref = "uiautomator" }
-apollo-runtime = { group = "com.apollographql.apollo3", name = "apollo-runtime", version.ref = "apolloGraphQL"}
-apollo-rx3-support = { group = "com.apollographql.apollo3", name = "apollo-rx3-support", version.ref = "apolloGraphQL"}
+apollo-runtime = { group = "com.apollographql.apollo3", name = "apollo-runtime", version.ref = "apolloGraphQL" }
+apollo-rx3-support = { group = "com.apollographql.apollo3", name = "apollo-rx3-support", version.ref = "apolloGraphQL" }
 
 [plugins]
 android-application = { id = "com.android.application", version.ref = "agp" }
 android-library = { id = "com.android.library", version.ref = "agp" }
 apollographql = { id = "com.apollographql.apollo3", version.ref = "apolloGraphQL" }
-vanniktech-maven-publish = { id = "com.vanniktech.maven.publish", version.ref = "vanniktechMavenPublish"}
+vanniktech-maven-publish = { id = "com.vanniktech.maven.publish", version.ref = "vanniktechMavenPublish" }

visual-espresso/visual/src/main/java/androidx/test/uiautomator/CustomAccessibilityNodeInfoDumper.java

@@ -0,0 +1,203 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.test.uiautomator;
+
+import android.util.Log;
+import android.util.Xml;
+import android.view.accessibility.AccessibilityNodeInfo;
+
+import org.xmlpull.v1.XmlSerializer;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Slightly modified version of {@link androidx.test.uiautomator.AccessibilityNodeInfoDumper}
+ * to include non-invisible elements in window hierarchy dump
+ */
+public class CustomAccessibilityNodeInfoDumper {
+
+    private static final String LOGTAG = CustomAccessibilityNodeInfoDumper.class.getSimpleName();
+    private static final String[] NAF_EXCLUDED_CLASSES = new String[]{
+            android.widget.GridView.class.getName(), android.widget.GridLayout.class.getName(),
+            android.widget.ListView.class.getName(), android.widget.TableLayout.class.getName()
+    };
+
+    public static void dumpWindowHierarchy(UiDevice device, OutputStream out) throws IOException {
+        XmlSerializer serializer = Xml.newSerializer();
+        serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
+        serializer.setOutput(out, "UTF-8");
+
+        serializer.startDocument("UTF-8", true);
+        serializer.startTag("", "hierarchy"); // TODO(allenhair): Should we use a namespace?
+        serializer.attribute("", "rotation", Integer.toString(device.getDisplayRotation()));
+
+        for (AccessibilityNodeInfo root : device.getWindowRoots()) {
+            dumpNodeRec(root, serializer, 0, device.getDisplayWidth(), device.getDisplayHeight());
+        }
+
+        serializer.endTag("", "hierarchy");
+        serializer.endDocument();
+    }
+
+    private static void dumpNodeRec(AccessibilityNodeInfo node, XmlSerializer serializer, int index,
+                                    int width, int height) throws IOException {
+        serializer.startTag("", "node");
+        if (!nafExcludedClass(node) && !nafCheck(node))
+            serializer.attribute("", "NAF", Boolean.toString(true));
+        serializer.attribute("", "index", Integer.toString(index));
+        serializer.attribute("", "text", safeCharSeqToString(node.getText()));
+        serializer.attribute("", "resource-id", safeCharSeqToString(node.getViewIdResourceName()));
+        serializer.attribute("", "class", safeCharSeqToString(node.getClassName()));
+        serializer.attribute("", "package", safeCharSeqToString(node.getPackageName()));
+        serializer.attribute("", "content-desc", safeCharSeqToString(node.getContentDescription()));
+        serializer.attribute("", "checkable", Boolean.toString(node.isCheckable()));
+        serializer.attribute("", "checked", Boolean.toString(node.isChecked()));
+        serializer.attribute("", "clickable", Boolean.toString(node.isClickable()));
+        serializer.attribute("", "enabled", Boolean.toString(node.isEnabled()));
+        serializer.attribute("", "focusable", Boolean.toString(node.isFocusable()));
+        serializer.attribute("", "focused", Boolean.toString(node.isFocused()));
+        serializer.attribute("", "scrollable", Boolean.toString(node.isScrollable()));
+        serializer.attribute("", "long-clickable", Boolean.toString(node.isLongClickable()));
+        serializer.attribute("", "password", Boolean.toString(node.isPassword()));
+        serializer.attribute("", "selected", Boolean.toString(node.isSelected()));
+        serializer.attribute("", "visible-to-user", Boolean.toString(node.isVisibleToUser()));
+        serializer.attribute("", "bounds", AccessibilityNodeInfoHelper.getVisibleBoundsInScreen(
+                node, width, height).toShortString());
+        int count = node.getChildCount();
+        for (int i = 0; i < count; i++) {
+            AccessibilityNodeInfo child = node.getChild(i);
+            if (child != null) {
+                dumpNodeRec(child, serializer, i, width, height);
+                child.recycle();
+            } else {
+                Log.i(LOGTAG, String.format("Null child %d/%d, parent: %s",
+                        i, count, node));
+            }
+        }
+        serializer.endTag("", "node");
+    }
+
+    /**
+     * The list of classes to exclude my not be complete. We're attempting to
+     * only reduce noise from standard layout classes that may be falsely
+     * configured to accept clicks and are also enabled.
+     *
+     * @param node
+     * @return true if node is excluded.
+     */
+    private static boolean nafExcludedClass(AccessibilityNodeInfo node) {
+        String className = safeCharSeqToString(node.getClassName());
+        for (String excludedClassName : NAF_EXCLUDED_CLASSES) {
+            if (className.endsWith(excludedClassName))
+                return true;
+        }
+        return false;
+    }
+
+    /**
+     * We're looking for UI controls that are enabled, clickable but have no
+     * text nor content-description. Such controls configuration indicate an
+     * interactive control is present in the UI and is most likely not
+     * accessibility friendly. We refer to such controls here as NAF controls
+     * (Not Accessibility Friendly)
+     *
+     * @param node
+     * @return false if a node fails the check, true if all is OK
+     */
+    private static boolean nafCheck(AccessibilityNodeInfo node) {
+        boolean isNaf = node.isClickable() && node.isEnabled()
+                && safeCharSeqToString(node.getContentDescription()).isEmpty()
+                && safeCharSeqToString(node.getText()).isEmpty();
+
+        if (!isNaf)
+            return true;
+
+        // check children since sometimes the containing element is clickable
+        // and NAF but a child's text or description is available. Will assume
+        // such layout as fine.
+        return childNafCheck(node);
+    }
+
+    /**
+     * This should be used when it's already determined that the node is NAF and
+     * a further check of its children is in order. A node maybe a container
+     * such as LinerLayout and may be set to be clickable but have no text or
+     * content description but it is counting on one of its children to fulfill
+     * the requirement for being accessibility friendly by having one or more of
+     * its children fill the text or content-description. Such a combination is
+     * considered by this dumper as acceptable for accessibility.
+     *
+     * @param node
+     * @return false if node fails the check.
+     */
+    private static boolean childNafCheck(AccessibilityNodeInfo node) {
+        int childCount = node.getChildCount();
+        for (int x = 0; x < childCount; x++) {
+            AccessibilityNodeInfo childNode = node.getChild(x);
+
+            if (!safeCharSeqToString(childNode.getContentDescription()).isEmpty()
+                    || !safeCharSeqToString(childNode.getText()).isEmpty())
+                return true;
+
+            if (childNafCheck(childNode))
+                return true;
+        }
+        return false;
+    }
+
+    private static String safeCharSeqToString(CharSequence cs) {
+        if (cs == null)
+            return "";
+        else {
+            return stripInvalidXMLChars(cs);
+        }
+    }
+
+    private static String stripInvalidXMLChars(CharSequence cs) {
+        StringBuffer ret = new StringBuffer();
+        char ch;
+        /* http://www.w3.org/TR/xml11/#charsets
+        [#x1-#x8], [#xB-#xC], [#xE-#x1F], [#x7F-#x84], [#x86-#x9F], [#xFDD0-#xFDDF],
+        [#x1FFFE-#x1FFFF], [#x2FFFE-#x2FFFF], [#x3FFFE-#x3FFFF],
+        [#x4FFFE-#x4FFFF], [#x5FFFE-#x5FFFF], [#x6FFFE-#x6FFFF],
+        [#x7FFFE-#x7FFFF], [#x8FFFE-#x8FFFF], [#x9FFFE-#x9FFFF],
+        [#xAFFFE-#xAFFFF], [#xBFFFE-#xBFFFF], [#xCFFFE-#xCFFFF],
+        [#xDFFFE-#xDFFFF], [#xEFFFE-#xEFFFF], [#xFFFFE-#xFFFFF],
+        [#x10FFFE-#x10FFFF].
+         */
+        for (int i = 0; i < cs.length(); i++) {
+            ch = cs.charAt(i);
+
+            if ((ch >= 0x1 && ch <= 0x8) || (ch >= 0xB && ch <= 0xC) || (ch >= 0xE && ch <= 0x1F) ||
+                    (ch >= 0x7F && ch <= 0x84) || (ch >= 0x86 && ch <= 0x9f) ||
+                    (ch >= 0xFDD0 && ch <= 0xFDDF) || (ch >= 0x1FFFE && ch <= 0x1FFFF) ||
+                    (ch >= 0x2FFFE && ch <= 0x2FFFF) || (ch >= 0x3FFFE && ch <= 0x3FFFF) ||
+                    (ch >= 0x4FFFE && ch <= 0x4FFFF) || (ch >= 0x5FFFE && ch <= 0x5FFFF) ||
+                    (ch >= 0x6FFFE && ch <= 0x6FFFF) || (ch >= 0x7FFFE && ch <= 0x7FFFF) ||
+                    (ch >= 0x8FFFE && ch <= 0x8FFFF) || (ch >= 0x9FFFE && ch <= 0x9FFFF) ||
+                    (ch >= 0xAFFFE && ch <= 0xAFFFF) || (ch >= 0xBFFFE && ch <= 0xBFFFF) ||
+                    (ch >= 0xCFFFE && ch <= 0xCFFFF) || (ch >= 0xDFFFE && ch <= 0xDFFFF) ||
+                    (ch >= 0xEFFFE && ch <= 0xEFFFF) || (ch >= 0xFFFFE && ch <= 0xFFFFF) ||
+                    (ch >= 0x10FFFE && ch <= 0x10FFFF))
+                ret.append(".");
+            else
+                ret.append(ch);
+        }
+        return ret.toString();
+    }
+}

visual-espresso/visual/src/main/java/com/saucelabs/visual/VisualApi.java

@@ -45,7 +45,14 @@ CreateSnapshotUploadMutation.Data uploadSnapshot(String buildId, boolean capture
         CreateSnapshotUploadMutation.Data data = graphQLClient.executeMutation(m);
 
         if (captureDom) {
-            byte[] dom = SnapshotHelper.getInstance().captureDom(clipElement);
+            byte[] dom;
+            if (clipElement != null) {
+                dom = SnapshotHelper.getInstance().captureDom(clipElement);
+            } else if (scrollView != null) {
+                dom = SnapshotHelper.getInstance().captureDom(scrollView);
+            } else {
+                dom = SnapshotHelper.getInstance().captureDom(null);
+            }
             SnapshotHelper.getInstance().uploadDom(data.result.domUploadUrl, dom);
         }
 

visual-espresso/visual/src/main/java/com/saucelabs/visual/utils/SnapshotHelper.java

@@ -8,12 +8,11 @@
 import android.text.TextUtils;
 import android.util.Base64;
 import android.view.View;
-import android.view.ViewGroup;
-import android.view.ViewParent;
 import android.widget.FrameLayout;
 import android.widget.TextView;
 
 import androidx.test.platform.app.InstrumentationRegistry;
+import androidx.test.uiautomator.CustomAccessibilityNodeInfoDumper;
 import androidx.test.uiautomator.UiDevice;
 
 import com.saucelabs.visual.exception.VisualApiException;
@@ -88,7 +87,7 @@ public byte[] captureDom(View clipElement) {
         String dom;
         UiDevice device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation());
         try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
-            device.dumpWindowHierarchy(os);
+            CustomAccessibilityNodeInfoDumper.dumpWindowHierarchy(device, os);
             dom = os.toString("UTF-8");
         } catch (IOException e) {
             throw new VisualApiException(e.getLocalizedMessage());
@@ -110,6 +109,7 @@ public byte[] captureDom(View clipElement) {
     /**
      * Uses resource id and text of the view (if available) to locate the view inside DOM
      * Can be extended further to include more fields
+     *
      * @param view View to be queried
      * @return Jsoup parseable query to locate the view
      */