Improved Semaphore CI configuration

.semaphore/semaphore.yml

@@ -46,26 +46,18 @@ global_job_config:
       - sudo rm -rf ~/.rbenv ~/.phpbrew
       - checkout
       # init environment variables
-      - export SUM=$(checksum build.gradle.kts)-$(checksum FlowCrypt/build.gradle.kts)-$(checksum ./script/ci-install-android-sdk.sh)
-      - export APP_GRADLE_CACHE=gradle-cache-$SUM # per conf files hash
-      - export ANDROID_SDK_CACHE=android-sdk-$SUM # per conf files hash
-      - export BUILD_CXX_CACHE=build-cxx-cache-$SEMAPHORE_GIT_BRANCH-$SUM  # per branch and conf files hash
-      - export BUILD_NATIVE_CACHE=build-native-cache-$SEMAPHORE_GIT_BRANCH-$SUM  # per branch and conf files hash
-      - export BUILD_CACHE=build-cache-$SEMAPHORE_GIT_BRANCH-$SUM  # per branch and conf files hash
+      - export GRADLE_SUM=$(checksum build.gradle.kts)-$(checksum FlowCrypt/build.gradle.kts)
+      - export SDK_SUM=$(checksum ./script/ci-install-android-sdk.sh)
+      - export APP_GRADLE_CACHE=project-gradle-cache-$GRADLE_SUM # project .gradle cache, per Gradle config hash
+      - export ANDROID_SDK_CACHE=android-sdk-$SDK_SUM # Android SDK cache, per SDK installer hash
       # restore app caches
       - cache restore $APP_GRADLE_CACHE
-      - cache restore $BUILD_CXX_CACHE
-      - cache restore $BUILD_NATIVE_CACHE
-      - cache restore $BUILD_CACHE
       # restore global caches
       - cache restore $ANDROID_SDK_CACHE
       - cache restore gradle-wrapper
-      - cache restore gradle-cache
-      - cache restore android-build-cache
+      - cache restore gradle-cache-$GRADLE_SUM
       # Install Android dependencies if needed
       - ./script/ci-install-android-sdk.sh
-      # Install ninja
-      - sudo apt install -y ninja-build
 blocks:
   - name: 'Build'
     execution_time_limit:
@@ -79,90 +71,25 @@ blocks:
             # print Java version
             - java -version
             # compile project
-            - ./gradlew --console=plain assembleConsumerUiTests
+            - ./gradlew --console=plain --no-daemon --build-cache assembleConsumerUiTests
       epilogue:
         on_pass:
           commands:
             # store app cache
             - echo "Store the app cache"
             - cache has_key $APP_GRADLE_CACHE || cache store $APP_GRADLE_CACHE .gradle
-            - cache has_key $BUILD_CXX_CACHE || cache store $BUILD_CXX_CACHE FlowCrypt/.cxx
-            - cache has_key $BUILD_NATIVE_CACHE || cache store $BUILD_NATIVE_CACHE FlowCrypt/.externalNativeBuild
-            - cache has_key $BUILD_CACHE || cache store $BUILD_CACHE FlowCrypt/build
 
             # clean and store global cache
             - echo "Store the global cache"
             - find ~/.gradle/caches/ -name "*.lock" -type f -delete # https://medium.com/cirruslabs/mastering-gradle-caching-and-incremental-builds-37eb1af7fcde
             - cache has_key $ANDROID_SDK_CACHE || cache store $ANDROID_SDK_CACHE $ANDROID_HOME
-            - cache delete gradle-wrapper
-            - cache delete gradle-cache
-            - cache delete android-build-cache
-            - cache store gradle-wrapper ~/.gradle/wrapper
-            - cache store gradle-cache ~/.gradle/caches
-            - cache store android-build-cache ~/.android/build-cache
+            - cache has_key gradle-wrapper || cache store gradle-wrapper ~/.gradle/wrapper
+            - cache has_key gradle-cache-$GRADLE_SUM || cache store gradle-cache-$GRADLE_SUM ~/.gradle/caches
 
   - name: 'Testing'
     task:
       jobs:
-        - name: 'Lint(structural quality)'
-          execution_time_limit:
-            minutes: 15
-          commands:
-            # run Lint checks
-            - ./script/ci-lint-checks.sh
-
-        - name: 'JUnit tests'
-          execution_time_limit:
-            minutes: 15
-          commands:
-            # run JUnit tests
-            - ./script/ci-junit-tests.sh
-
-        - name: 'Instrumentation tests(No email server)'
-          execution_time_limit:
-            minutes: 60
-          matrix:
-            - env_var: EMULATOR
-              values: [ "0", "1", "2", "3" ]
-          commands:
-            # Setup and run an emulator
-            - ./script/ci-setup-and-run-emulator.sh
-            # wait until ready
-            - ./script/ci-wait-for-emulator.sh
-            # Run filtered logging for TestRunner
-            - adb logcat -v color TestRunner:V *:S > ~/logcat_log.txt &
-            # Run instrumentation tests
-            - ./script/ci-instrumentation-tests-without-mailserver.sh 4 $EMULATOR
-
-        - name: 'Instrumentation tests(with email server)'
-          execution_time_limit:
-            minutes: 60
-          commands:
-            # Run an email server
-            - cd docker-mailserver && ./run_email_server.sh && cd -
-            # Setup and run an emulator
-            - ./script/ci-setup-and-run-emulator.sh
-            #wait until ready
-            - ./script/ci-wait-for-emulator.sh
-            # Run filtered logging for TestRunner
-            - adb logcat -v color TestRunner:V *:S > ~/logcat_log.txt &
-            # Run instrumentation tests
-            - ./script/ci-instrumentation-tests-with-mailserver.sh
-
-        - name: 'Instrumentation tests(enterprise)'
-          execution_time_limit:
-            minutes: 60
-          commands:
-            # Setup and run an emulator
-            - ./script/ci-setup-and-run-emulator.sh
-            #wait until ready
-            - ./script/ci-wait-for-emulator.sh
-            # Run filtered logging for TestRunner
-            - adb logcat -v color TestRunner:V *:S > ~/logcat_log.txt &
-            # Run instrumentation tests
-            - ./script/ci-instrumentation-tests-enterprise.sh
-
-        - name: 'Instrumentation tests(flaky)'
+        - name: 'Instrumentation tests(debug)'
           execution_time_limit:
             minutes: 60
           commands:
@@ -188,6 +115,8 @@ blocks:
           commands:
             # collect and store debug info as artifacts
             - ./script/ci-get-and-publish-debug-info-as-artifact.sh
+            # print debug info
+            - ./script/ci-after-fail-debug.sh
 
 after_pipeline:
   task:

FlowCrypt/build.gradle.kts

@@ -6,8 +6,6 @@
 
 import com.android.build.api.artifact.SingleArtifact
 import com.android.build.api.variant.ResValue
-import org.gradle.api.GradleException
-import java.io.File
 import com.android.ddmlib.DdmPreferences
 import java.io.FileInputStream
 import java.text.SimpleDateFormat
@@ -127,10 +125,10 @@ android {
         "SHARED_TENANT_FES_URL",
         "\"https://flowcrypt.test/shared-tenant-fes/\""
       )
-      buildConfigField("boolean", "IS_HTTP_LOG_ENABLED", "false")
-      buildConfigField("String", "HTTP_LOG_LEVEL", "\"NONE\"")
-      resValue("string", "gradle_is_http_log_enabled", "false")
-      resValue("string", "gradle_http_log_level", "NONE")
+      buildConfigField("boolean", "IS_HTTP_LOG_ENABLED", "true")
+      buildConfigField("String", "HTTP_LOG_LEVEL", "\"BODY\"")
+      resValue("string", "gradle_is_http_log_enabled", "true")
+      resValue("string", "gradle_http_log_level", "BODY")
     }
   }
 

FlowCrypt/src/androidTest/java/com/flowcrypt/email/ui/gmailapi/SearchMessagesGmailApiFlowTest.kt

@@ -20,6 +20,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.filters.MediumTest
 import com.flowcrypt.email.R
 import com.flowcrypt.email.TestConstants
+import com.flowcrypt.email.junit.annotations.DebugTest
 import com.flowcrypt.email.junit.annotations.FlowCryptTestSettings
 import com.flowcrypt.email.matchers.CustomMatchers.Companion.withRecyclerViewItemCount
 import com.flowcrypt.email.rules.ClearAppSettingsRule
@@ -49,6 +50,7 @@ import java.util.concurrent.TimeUnit
 @MediumTest
 @RunWith(AndroidJUnit4::class)
 @FlowCryptTestSettings(useCommonIdling = false)
+@DebugTest
 class SearchMessagesGmailApiFlowTest : BaseGmailApiTest() {
 
   override val mockWebServerRule =

docker/TestEnvironment/Dockerfile

@@ -0,0 +1,42 @@
+FROM ubuntu:24.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG ANDROID_PLATFORM=android-36
+ARG ANDROID_SYSTEM_IMAGE=system-images;android-36;google_apis;x86_64
+ARG ANDROID_BUILD_TOOLS=36.0.0
+
+ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64
+ENV ANDROID_HOME=/opt/android-sdk
+ENV ANDROID_SDK_ROOT=/opt/android-sdk
+ENV ANDROID_AVD_HOME=/opt/android-avd
+ENV PATH=/opt/android-sdk/emulator:/opt/android-sdk/platform-tools:/opt/android-sdk/cmdline-tools/latest/bin:/usr/lib/jvm/java-21-openjdk-amd64/bin:${PATH}
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash \
+    ca-certificates \
+    curl \
+    openjdk-21-jdk \
+    unzip \
+    wget \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY script/ci-install-android-sdk.sh /tmp/ci-install-android-sdk.sh
+RUN chmod +x /tmp/ci-install-android-sdk.sh \
+    && mkdir -p "${ANDROID_AVD_HOME}" \
+    && ANDROID_PLATFORM="${ANDROID_PLATFORM}" \
+      ANDROID_SYSTEM_IMAGE="${ANDROID_SYSTEM_IMAGE}" \
+      ANDROID_BUILD_TOOLS="${ANDROID_BUILD_TOOLS}" \
+      INSTALL_KVM_DEPS=0 \
+      RUN_KVM_CHECK=0 \
+      /tmp/ci-install-android-sdk.sh \
+    && sdkmanager --list_installed \
+    && rm -f /tmp/ci-install-android-sdk.sh
+
+RUN mkdir -p /opt/flowcrypt/scripts
+COPY script/create-avd.sh /opt/flowcrypt/scripts/create-avd.sh
+COPY script/run-emulator.sh /opt/flowcrypt/scripts/run-emulator.sh
+RUN chmod +x /opt/flowcrypt/scripts/create-avd.sh /opt/flowcrypt/scripts/run-emulator.sh
+
+WORKDIR /workspace
+
+CMD ["/bin/bash"]

docker/TestEnvironment/README.md

@@ -0,0 +1,54 @@
+# Test Environment Docker Image
+
+This directory contains a Docker context for a headless Android test environment based on
+`ubuntu:24.04`.
+
+The image installs:
+
+- OpenJDK 21
+- Android SDK command-line tools
+- `platform-tools`
+- Android Emulator
+- Android platform `android-36`
+- Build tools `36.0.0`
+- System image `system-images;android-36;google_apis;x86_64`
+
+## Build
+
+```bash
+docker build -t flowcrypt/android-test-env -f docker/TestEnvironment/Dockerfile .
+```
+
+## Create an AVD inside the container
+
+```bash
+docker run --rm -it flowcrypt/android-test-env /opt/flowcrypt/scripts/create-avd.sh
+```
+
+## Run the emulator inside the container
+
+The container includes reusable helpers:
+
+- `/opt/flowcrypt/scripts/create-avd.sh`
+- `/opt/flowcrypt/scripts/run-emulator.sh`
+
+The container includes the emulator binaries, but actual emulator launch usually needs:
+
+- `/dev/kvm` passed through to the container
+- additional Docker flags such as `--device /dev/kvm`
+
+Example:
+
+```bash
+docker run --rm -it \
+  --device /dev/kvm \
+  flowcrypt/android-test-env \
+  bash
+```
+
+Then inside the container:
+
+```bash
+/opt/flowcrypt/scripts/create-avd.sh
+/opt/flowcrypt/scripts/run-emulator.sh
+```

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,11 @@
+#
+# © 2016-present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com
+# Contributors: denbond7
+#
+
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.5.0-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

script/ci-after-fail-debug.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+#
+# © 2016-present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com
+# Contributors: denbond7
+#
+
+set -euo pipefail
+set -o xtrace
+
+check_ping_or_fail() {
+  local host="$1"
+  local label="$2"
+
+  for attempt in {1..30}; do
+    if adb shell "ping -c 1 ${host}"; then
+      return 0
+    fi
+
+    if [[ "$attempt" -eq 30 ]]; then
+      echo "Failed to ping ${host}: ${label}"
+      exit 1
+    fi
+
+    sleep 2
+  done
+}
+
+###################################################################################################
+check_ping_or_fail "www.google.com" "internet connection"
+check_ping_or_fail "fes.flowcrypt.test" "flowcrypt.test forwarding"
+
+set +o xtrace

script/ci-after-success-actions-for-instrumentation-tests.sh

@@ -1,18 +1,30 @@
-#!/bin/bash
+#!/usr/bin/env bash
+
+#
+# © 2016-present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com
+# Contributors: denbond7
+#
+
+set -euo pipefail
 
 if [[ "$SEMAPHORE_JOB_NAME" =~ ^Instrumentation.* ]]; then
     # print debug info about connected device
     echo "Print connected devices"
     adb devices
 
-    # store logcat log
+  if [[ -f "$HOME/logcat_log.txt" ]]; then
     echo "Store logcat log"
-    artifact push job ~/logcat_log.txt
+    artifact push job "$HOME/logcat_log.txt"
+  else
+    echo "No logcat_log.txt found, skipping"
+  fi
 
-    # store screenshots
-    echo "Store screenshots"
-    adb pull "/sdcard/Pictures"
-    adb shell ls /sdcard/Pictures
+  echo "Store screenshots"
+  if adb shell test -d /sdcard/Pictures; then
+    rm -rf Pictures
+    adb pull "/sdcard/Pictures" Pictures
     artifact push job Pictures
+  else
+    echo "No /sdcard/Pictures directory found, skipping"
+  fi
 fi
-