Merge PR #1292.

dart_test.yaml

@@ -0,0 +1,17 @@
+# define available tags
+tags:
+    # Tests which we want to run on desktop
+    desktop:
+
+    # Tests which we want to run on Android
+    android:
+
+    # Tests consuming quiet a lot of time
+    slow:
+
+    # Minimal tests
+    # quick verification that the framework is working
+    minimal:
+
+    # OATH tests
+    oath:

integration_test/keyless_test.dart

@@ -14,10 +14,13 @@
  * limitations under the License.
  */
 
+@Tags(['desktop', 'android'])
+
 import 'package:flutter/material.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:integration_test/integration_test.dart';
 import 'package:yubico_authenticator/app/views/keys.dart';
+import 'package:yubico_authenticator/core/state.dart';
 
 import 'utils/keyless_test_util.dart';
 import 'utils/test_util.dart';
@@ -26,6 +29,10 @@ void main() {
   var binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized();
   binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
 
+  group('Startup', () {
+    appTestKeyless('App starts', (WidgetTester tester) async {},
+        tags: 'minimal');
+  });
   group('Settings', () {
     appTestKeyless('Click through all Themes', (WidgetTester tester) async {
       var settingDrawerButton = find.byKey(settingDrawerIcon).hitTestable();
@@ -64,30 +71,47 @@ void main() {
       appTestKeyless('TOS link', (WidgetTester tester) async {
         await tester.tap(helpDrawerButton);
         await tester.longWait();
-        await tester.tap(find.byKey(tosButton).hitTestable());
-        await tester.longWait();
+        if (isAndroid) {
+          expect(find.byKey(tosButton).hitTestable(), findsOneWidget);
+        } else {
+          await tester.tap(find.byKey(tosButton).hitTestable());
+          await tester.longWait();
+        }
       });
       appTestKeyless('Privacy link', (WidgetTester tester) async {
         await tester.tap(helpDrawerButton);
         await tester.longWait();
-        await tester.tap(find.byKey(privacyButton).hitTestable());
-        await tester.longWait();
+        if (isAndroid) {
+          expect(find.byKey(privacyButton).hitTestable(), findsOneWidget);
+        } else {
+          await tester.tap(find.byKey(privacyButton).hitTestable());
+          await tester.longWait();
+        }
       });
       appTestKeyless('Feedback link', (WidgetTester tester) async {
         await tester.tap(helpDrawerButton);
         await tester.longWait();
-        await tester.tap(find.byKey(feedbackButton).hitTestable());
-        await tester.longWait();
+        if (isAndroid) {
+          expect(find.byKey(feedbackButton).hitTestable(), findsOneWidget);
+        } else {
+          await tester.tap(find.byKey(feedbackButton).hitTestable());
+          await tester.longWait();
+        }
       });
       appTestKeyless('Help link', (WidgetTester tester) async {
         await tester.tap(helpDrawerButton);
         await tester.longWait();
-        await tester.tap(find.byKey(helpButton).hitTestable());
-        await tester.longWait();
+        if (isAndroid) {
+          expect(find.byKey(helpButton).hitTestable(), findsOneWidget);
+        } else {
+          await tester.tap(find.byKey(helpButton).hitTestable());
+          await tester.longWait();
+        }
       });
     });
     group('Troubleshooting', () {
-      appTestKeyless('Diagnostics Button', (WidgetTester tester) async {
+      appTestKeyless('Diagnostics Button', skip: isAndroid,
+          (WidgetTester tester) async {
         await tester.tap(helpDrawerButton);
         await tester.longWait();
         await tester.tap(find.byKey(diagnosticsChip).hitTestable());

integration_test/oath_test.dart

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+@Tags(['android', 'desktop', 'oath'])
 import 'package:flutter_test/flutter_test.dart';
 import 'package:integration_test/integration_test.dart';
 import 'package:yubico_authenticator/app/views/keys.dart';
@@ -25,20 +26,6 @@ import 'package:yubico_authenticator/oath/views/account_list.dart';
 import 'utils/oath_test_util.dart';
 import 'utils/test_util.dart';
 
-String randomPadded() {
-  return randomNum(999).toString().padLeft(3, '0');
-}
-
-randomNum(int i) {}
-
-String generateRandomIssuer() {
-  return 'i${randomPadded()}';
-}
-
-String generateRandomName() {
-  return 'n${randomPadded()}';
-}
-
 void main() {
   var binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized();
   binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
@@ -59,16 +46,12 @@ void main() {
   group('Account creation', () {
     appTest('Initial reset OATH', (WidgetTester tester) async {
       /// reset OATH application
-      var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
-      await tester.tap(oathDrawerButton);
-      await tester.longWait();
+      await tester.tapAppDrawerButton(oathAppDrawer);
       await tester.resetOATH();
       await tester.longWait();
     });
     appTest('Create 32 Accounts', (WidgetTester tester) async {
-      var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
-      await tester.tap(oathDrawerButton);
-      await tester.longWait();
+      await tester.tapAppDrawerButton(oathAppDrawer);
 
       for (var i = 0; i < 32; i += 1) {
         // just now merely 32 accounts
@@ -91,14 +74,12 @@ void main() {
       // TODO: verify one more addAccount() is not possible
       await tester.resetOATH();
       await tester.shortWait();
-    });
+    }, tags: ['slow']);
     // appTest('Create weird character-accounts and check byte count',
     //     (WidgetTester tester) async {});
     group('TOTP account tests', () {
       appTest('TOTP: sha-1', (WidgetTester tester) async {
-        var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
-        await tester.tap(oathDrawerButton);
-        await tester.longWait();
+        await tester.tapAppDrawerButton(oathAppDrawer);
         const testAccount = Account(
             issuer: 'i_totp_sha1',
             name: 'n__totp_sha1',
@@ -116,9 +97,7 @@ void main() {
         await tester.shortWait();
       });
       appTest('TOTP: sha-256', (WidgetTester tester) async {
-        var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
-        await tester.tap(oathDrawerButton);
-        await tester.longWait();
+        await tester.tapAppDrawerButton(oathAppDrawer);
         const testAccount = Account(
             issuer: 'i_totp_sha256',
             name: 'n__totp_sha256',
@@ -136,9 +115,7 @@ void main() {
         await tester.shortWait();
       });
       appTest('TOTP: sha-512', (WidgetTester tester) async {
-        var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
-        await tester.tap(oathDrawerButton);
-        await tester.longWait();
+        await tester.tapAppDrawerButton(oathAppDrawer);
         const testAccount = Account(
             issuer: 'i_totp_sha512',
             name: 'n__totp_sha512',
@@ -164,9 +141,7 @@ void main() {
       // appTest('TOTP: digits-8',
       //     (WidgetTester tester) async {});
       appTest('TOTP: touch', (WidgetTester tester) async {
-        var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
-        await tester.tap(oathDrawerButton);
-        await tester.longWait();
+        await tester.tapAppDrawerButton(oathAppDrawer);
         const testAccount = Account(
             issuer: 'i_totp_touch',
             name: 'n_totp_touch',
@@ -185,9 +160,7 @@ void main() {
     });
     // group('HOTP account tests', () {
     appTest('HOTP: sha-1', (WidgetTester tester) async {
-      var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
-      await tester.tap(oathDrawerButton);
-      await tester.longWait();
+      await tester.tapAppDrawerButton(oathAppDrawer);
       const testAccount = Account(
           issuer: 'i_hotp_sha1',
           name: 'n__hotp_sha1',
@@ -205,9 +178,7 @@ void main() {
       await tester.shortWait();
     });
     appTest('HOTP: sha-256', (WidgetTester tester) async {
-      var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
-      await tester.tap(oathDrawerButton);
-      await tester.longWait();
+      await tester.tapAppDrawerButton(oathAppDrawer);
       const testAccount = Account(
           issuer: 'i_hotp_sha256',
           name: 'n__hotp_sha256',
@@ -225,9 +196,7 @@ void main() {
       await tester.shortWait();
     });
     appTest('HOTP: sha-512', (WidgetTester tester) async {
-      var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
-      await tester.tap(oathDrawerButton);
-      await tester.longWait();
+      await tester.tapAppDrawerButton(oathAppDrawer);
       const testAccount = Account(
           issuer: 'i_hotp_sha512',
           name: 'n__hotp_sha512',
@@ -247,9 +216,7 @@ void main() {
     // appTest('TOTP: digits-8',
     //     (WidgetTester tester) async {});
     appTest('HOTP: touch', (WidgetTester tester) async {
-      var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
-      await tester.tap(oathDrawerButton);
-      await tester.longWait();
+      await tester.tapAppDrawerButton(oathAppDrawer);
       const testAccount = Account(
           issuer: 'i_hotp_touch',
           name: 'n_hotp_touch',
@@ -268,9 +235,7 @@ void main() {
     // group('QR Code scanning', () {});
     appTest('Final reset OATH', (WidgetTester tester) async {
       /// reset OATH application
-      var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
-      await tester.tap(oathDrawerButton);
-      await tester.longWait();
+      await tester.tapAppDrawerButton(oathAppDrawer);
       await tester.resetOATH();
       await tester.longWait();
     });
@@ -284,8 +249,9 @@ void main() {
       await tester.deleteAccount(testAccount);
       await tester.deleteAccount(
           const Account(issuer: 'RenamedIssuer', name: 'RenamedName'));
-
+      await tester.longWait();
       await tester.addAccount(testAccount);
+      await tester.longWait();
       await tester.renameAccount(testAccount, 'RenamedIssuer', 'RenamedName');
     });
   });

integration_test/testrunner-android.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+#
+# Copyright (C) 2023 Yubico.
+#
+# This file defines which tests we should run in the CI environment
+# It is now being used to check for flakiness, as we haven't decided
+# which tests will be run in CI.
+
+if (( $# < 1 )); then
+    echo "Usage $(basename $0) DEVICE_ID [TAGS]"
+    exit 1
+fi
+
+DEVICE="${1}"
+
+if (( $# < 2 )); then
+  TAGS="android" # default
+else
+  TAGS="(${2}) && android"
+fi
+
+echo "Running tests matching tag expression: $TAGS"
+
+ANDROID_TESTS=('integration_test/oath_test.dart' 'integration_test/keyless_test.dart')
+
+DRIVER="integration_test/utils/android/test_driver.dart"
+
+flutter test \
+  --tags "${TAGS}" \
+  --device-id "${DEVICE}" \
+  --no-pub \
+  --no-track-widget-creation \
+  --reporter compact \
+  --file-reporter "github:build/integration_test_run_$(date +'%Y%m%d_%H:%M:%S')" \
+  "${ANDROID_TESTS[@]}"

integration_test/utils/keyless_test_util.dart

@@ -44,7 +44,8 @@ extension AppWidgetTester on WidgetTester {
     }
 
     if (evaluated.isEmpty) {
-      testLog(false, 'Failed to find ${f.evaluate} in $timeOutSec seconds.');
+      testLog(false,
+          'Found 0 ${f.describeMatch(Plurality.zero)} in $timeOutSec seconds.');
     }
 
     return f;
@@ -120,9 +121,10 @@ void appTestKeyless(
   WidgetTesterCallback callback, {
   bool? skip,
   Map startUpParams = const {},
+  dynamic tags,
 }) {
-  testWidgets(description, (WidgetTester tester) async {
+  testWidgets(description, skip: skip, (WidgetTester tester) async {
     await tester.startUp(startUpParams);
     await callback(tester);
-  });
+  }, tags: tags);
 }

integration_test/utils/oath_test_util.dart

@@ -59,8 +59,10 @@ extension OathFunctions on WidgetTester {
     await longWait();
     await tap(find.byKey(keys.addAccountAction).hitTestable());
     await longWait();
-    await tap(find.byKey(keys.addAccountManuallyButton).hitTestable());
-    await longWait();
+    if (isDesktop) {
+      await tap(find.byKey(keys.addAccountManuallyButton).hitTestable());
+      await longWait();
+    }
   }
 
   Future<void> addAccount(Account a, {bool quiet = true}) async {
@@ -88,6 +90,10 @@ extension OathFunctions on WidgetTester {
     await tap(secretText);
     await enterText(secretText, a.secret);
     await shortWait();
+    if (isAndroid) {
+      FocusManager.instance.primaryFocus?.unfocus();
+      await shortWait();
+    }
     if (a.touch != null && a.touch == true) {
       var requireTouchFilterChip =
           find.byKey(keys.requireTouchFilterChip).hitTestable();

integration_test/utils/test_util.dart

@@ -108,6 +108,18 @@ extension AppWidgetTester on WidgetTester {
   bool isDrawerOpened() =>
       hasDrawer() == false || scaffoldGlobalKey.currentState!.isDrawerOpen;
 
+  /// Tap a app button in the drawer
+  /// If the drawer is closed, it is opened first
+  Future<void> tapAppDrawerButton(Key appKey) async {
+    if (hasDrawer() && !isDrawerOpened()) {
+      await openDrawer();
+    }
+
+    var appButtonFinder = find.byKey(appKey).hitTestable();
+    await tap(appButtonFinder);
+    await longWait();
+  }
+
   /// Management screen
   Future<void> openManagementScreen() async {
     if (!isDrawerOpened()) {
@@ -242,9 +254,10 @@ void appTest(
   WidgetTesterCallback callback, {
   bool? skip,
   Map startUpParams = const {},
+  dynamic tags,
 }) {
-  testWidgets(description, (WidgetTester tester) async {
+  testWidgets(description, skip: skip, (WidgetTester tester) async {
     await tester.startUp(startUpParams);
     await callback(tester);
-  });
+  }, tags: tags);
 }