Cleaning up some documentation.

doc/Development.adoc

@@ -90,7 +90,7 @@ These do not require a YubiKey, and are relatively quick to run.
 The integration tests are slower but cover more end-to-end functionality. The
 require an attached YubiKey to run, and will make modifications to the data
 stored on that YubiKey. For instructions on running these tests, see
-link:Integration_Tests.adoc[these instructions].
+link:../integration_test/testdoc.adoc[these instructions].
 
 
 === Packaging for MacOS

integration_test/keyless_test.dart

@@ -22,7 +22,6 @@ import 'package:yubico_authenticator/app/views/keys.dart';
 import 'utils/keyless_test_util.dart';
 import 'utils/test_util.dart';
 
-/// TODO: These need to be able to run keyless to run in CI.
 void main() {
   var binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized();
   binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;

integration_test/oath_test.dart

@@ -78,7 +78,6 @@ void main() {
       await tester.tap(oathDrawerButton);
       await tester.longWait();
 
-      /// TODO change back to 32 after flakiness eval
       for (var i = 0; i < 32; i += 1) {
         // just now merely 32 accounts
         var testAccount = Account(
@@ -104,39 +103,6 @@ void main() {
     // appTest('Create weird character-accounts and check byte count',
     //     (WidgetTester tester) async {});
     group('TOTP account tests', () {
-      // appTest('Create regular TOTP account', (WidgetTester tester) async {
-      //   // account with issuer field
-      //   // var issuer = generateRandomIssuer();
-      //   // var name = generateRandomName();
-      //   // var secret = 'abcdabcd';
-      //   var testAccount = const Account(
-      //     issuer: 'IssuerForTests',
-      //     name: 'NameForTests',
-      //     secret: 'abcdabcd',
-      //   );
-      //   var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
-      //   await tester.tap(oathDrawerButton);
-      //   await tester.longWait();
-      //
-      //   await tester.addAccount(testAccount);
-      //   await tester.longWait();
-      //
-      //   // TODO: Verify account exists
-      //   // TODO: Change testAccount
-      //   await tester.deleteAccount(testAccount);
-      // });
-      //
-      // appTest('Create issuer-less TOTP account', (WidgetTester tester) async {
-      //   // account without issuer field
-      //   var testAccount = const Account(
-      //     name: 'NoIssuerName',
-      //     secret: 'bbbbbbbbbbbbbbbb',
-      //   );
-      //   await tester.deleteAccount(testAccount);
-      //
-      //   /// TODO: change issuer functionality in oath_test_util
-      //   await tester.addAccount(testAccount);
-      // });
       appTest('TOTP: sha-1', (WidgetTester tester) async {
         var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable();
         await tester.tap(oathDrawerButton);

integration_test/piv_test.dart

@@ -20,7 +20,6 @@ import 'package:integration_test/integration_test.dart';
 import 'package:yubico_authenticator/app/views/keys.dart';
 import 'package:yubico_authenticator/core/state.dart';
 import 'package:yubico_authenticator/piv/keys.dart';
-// import 'package:yubico_authenticator/widgets/tooltip_if_truncated.dart';
 
 import 'utils/piv_test_util.dart';
 import 'utils/test_util.dart';
@@ -32,9 +31,6 @@ void main() {
   group('PIV Settings', skip: isAndroid, () {
     const factoryPin = '123456';
     const factoryPuk = '12345678';
-    // TODO: use or remove factoryManagementKey
-    // const factoryManagemenKey =
-    //    '010203040506070801020304050607080102030405060708';
     appTest('Reset PIV (settings-init)', (WidgetTester tester) async {
       await tester.resetPiv();
       await tester.shortWait();
@@ -48,7 +44,7 @@ void main() {
       await tester.shortWait();
       await tester.tap(find.byKey(actionsIconButtonKey).hitTestable());
 
-      /// TODO: This expect needs to verify that Pin underline is 'Blocked'
+      /// TODO: This expect needs to verify that Pin subtitle is 'Blocked'
       /// expect(find.byKey(managePinAction), find.byTooltip('Blocked'));
       await tester.shortWait();
       await tester.tap(find.byKey(managePinAction).hitTestable());
@@ -177,9 +173,7 @@ void main() {
         await tester.enterText(
             find.byKey(newPinPukField).hitTestable(), shortmanagementkey);
         await tester.longWait();
-        await tester.tap(find.byKey(saveButton).hitTestable());
+        expect(tester.isTextButtonEnabled(saveButton), false);
-        await tester.longWait();
-        // TODO: verify state
       });
       appTest('Change managementkey key', (WidgetTester tester) async {
         await tester.configurePiv();
@@ -260,7 +254,6 @@ void main() {
         await tester.resetPiv();
       });
 
-      /// TODO: The rest of management key settings, when input fields are fixed
       appTest('Reset PIV (settings-exit)', (WidgetTester tester) async {
         await tester.resetPiv();
         await tester.shortWait();
@@ -367,16 +360,16 @@ void main() {
       await tester.longWait();
       // 9 Verify Subject, verify Date
       //      TODO: this seems not to work!
-/*      expect(find.byWidgetPredicate((widget) {
+      // expect(find.byWidgetPredicate((widget) {
-        if (widget is TooltipIfTruncated) {
+      //   if (widget is TooltipIfTruncated) {
-          final TooltipIfTruncated textWidget = widget;
+      //     final TooltipIfTruncated textWidget = widget;
-          if (textWidget.key == certInfoSubjectKey &&
+      //     if (textWidget.key == certInfoSubjectKey &&
-              textWidget.text == 'CN=foobar') {
+      //         textWidget.text == 'CN=Generate9c') {
-            return true;
+      //       return true;
-          }
+      //     }
-        }
+      //   }
-        return false;
+      //   return false;
-      }), findsOneWidget);*/
+      // }), findsOneWidget);
 
       await tester.longWait();
       // 10. Delete Certificate
@@ -491,58 +484,14 @@ void main() {
       await tester.tap(find.byKey(deleteButton).hitTestable());
       await tester.longWait();
     });
-    // appTest('Import outdated Key+Certificate from file',
+    appTest('Import outdated Key+Certificate from file',
-    //     (WidgetTester tester) async {
+        (WidgetTester tester) async {});
-    //   /// TODO fileload needs to be handled
+
-    //   // 1. open PIV view
+    /// TODO fileload needs to be handled
-    //   var pivDrawerButton = find.byKey(pivAppDrawer).hitTestable();
+    appTest('Import neverexpire Key+Certificate from file',
-    //   await tester.tap(pivDrawerButton);
+        (WidgetTester tester) async {
-    //   await tester.longWait();
+      /// TODO fileload needs to be handled
-    //   // 2. click meatball menu for 9c
+    });
-    //   await tester.tap(find.byKey(meatballButton9c).hitTestable());
-    //   await tester.longWait();
-    //   // 3. click import
-    //   await tester.tap(find.byKey(importAction).hitTestable());
-    //   await tester.longWait();
-    //   // 4. pick key: outdated_key.pem and "Choose"
-    //   // 5. TODO: tap close
-    //   // 6. Verify slot 9c "Key without certificate loaded"
-    //   // 7. click meatball menu for 9c
-    //   await tester.tap(find.byKey(meatballButton9c).hitTestable());
-    //   await tester.longWait();
-    //   // 8. click import
-    //   await tester.tap(find.byKey(importAction).hitTestable());
-    //   await tester.longWait();
-    //   // 9. pick key: outdated_cert.pem and "Choose"
-    //   // 10. Tap "Import" on 'Import File Dialogue'
-    //   // Verify Certificate
-    // });
-    // appTest('Import neverexpire Key+Certificate from file',
-    //     (WidgetTester tester) async {
-    //   /// TODO fileload needs to be handled
-    //   // // 1. open PIV view
-    //   // var pivDrawerButton = find.byKey(pivAppDrawer).hitTestable();
-    //   // await tester.tap(pivDrawerButton);
-    //   // await tester.longWait();
-    //   // // 2. click meatball menu for 9d
-    //   // await tester.tap(find.byKey(meatballButton9d).hitTestable());
-    //   // await tester.longWait();
-    //   // // 3. click import
-    //   // await tester.tap(find.byKey(importAction).hitTestable());
-    //   // await tester.longWait();
-    //   // // 4. pick key: neverexpire_key.pem and "Choose"
-    //   // // 5. TODO: tap close
-    //   // // 6. Verify slot 9c "Key without certificate loaded"
-    //   // // 7. click meatball menu for 9d
-    //   // await tester.tap(find.byKey(meatballButton9d).hitTestable());
-    //   // await tester.longWait();
-    //   // // 8. click import
-    //   // await tester.tap(find.byKey(importAction).hitTestable());
-    //   // await tester.longWait();
-    //   // // 9. pick key: neverexpire_cert.pem and "Choose"
-    //   // // 10. Tap "Import" on 'Import File Dialogue'
-    //   // // Verify Certificate
-    // });
     appTest('Generate a CSR', (WidgetTester tester) async {
       // 1. open PIV view
       var pivDrawerButton = find.byKey(pivAppDrawer).hitTestable();

integration_test/testdoc.adoc

@@ -1,8 +1,10 @@
 == Testing Yubico Authenticator
 
-We have set up testing of the Yubico Authenticator flutter as a part of one of the following groups.
+Verifying the quality of Yubico Authenticator is made through multiple levels of tests, loosely
+defined in the following groups.
 
 * Automatic tests in CI pipeline
+* Unit tests
 * Automatic tests requiring a Yubikey
 * Manual tests
 
@@ -12,7 +14,7 @@ as they depend on other technology which are a bit to expensive to automate (cam
 the flutter test framework.
 
 To run the tests you need to specify the serial number of a Yubikey which the tests are able to run
-on (OBS this key will be reset and by running the tests).
+on (OBS this key will be reset by running the tests, so don't use one of your personal keys).
 
 === Desktop
 Running the tests for the CI environment:
@@ -21,9 +23,18 @@ Running the tests for the CI environment:
 
 Running all the tests for desktop (requires a Yubikey):
 
-    $ flutter test integration_test
+    $ ./testrunner.sh
 
-We will have this handled through a shell interface, more info to come.
+Running all the tests for desktop (requires a Yubikey):
+
+    $ flutter test
+
+The testrunner interface is work in progress, but it deals with a problem of the flutter
+integration_test framework, i.e. that it fails to initiate multiple UI-tests.
+
+Manual test scripts will be moved here so all quality related information is in the same place.
 
 === Android
-TBD
+
+Testing of GUI of the Android client is also a work in progress as tests work differently on the
+mobile platform.

integration_test/utils/keyless_test_util.dart

@@ -29,7 +29,6 @@ String? yubiKeyFirmware;
 String? yubiKeySerialNumber;
 bool collectedYubiKeyInformation = false;
 
-/// TODO: clean up this monster of appTestKeyLess
 extension AppWidgetTester on WidgetTester {
   /// waits up to [timeOutSec] seconds evaluating whether [Finder] f is
   /// visible

integration_test/utils/oath_test_util.dart

@@ -14,9 +14,6 @@
  * limitations under the License.
  */
 
-import 'dart:convert';
-import 'dart:math';
-
 import 'package:flutter/material.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:yubico_authenticator/app/views/keys.dart' as app_keys;
@@ -29,38 +26,6 @@ import 'package:yubico_authenticator/oath/views/account_view.dart';
 import '../utils/test_util.dart';
 import 'android/util.dart';
 
-/// THESE SHOULD PROBABLY BE REMOVOVED:
-///
-String randomPadded() {
-  return randomNum(999).toString().padLeft(3, '0');
-}
-
-randomNum(int i) {}
-
-String generateRandomIssuer() {
-  final random = Random.secure();
-  return 'issuer_${base64Encode(List.generate(4, (_) => random.nextInt(256)))}';
-  // return 'i${randomPadded()}';
-}
-
-String generateRandomName() {
-  final random = Random.secure();
-  return 'name_${base64Encode(List.generate(4, (_) => random.nextInt(256)))}';
-  //return 'n${randomPadded()}';
-}
-
-String generateRandomSecret() {
-  final random = Random.secure();
-  return base64Encode(List.generate(8, (_) => random.nextInt(256)));
-}
-
-String staticSecret() {
-  return 'abba';
-}
-
-///
-/// THESE SHOULD PROBABLY BE REMOVOVED
-
 class Account {
   final String? issuer;
   final String name;
@@ -68,6 +33,7 @@ class Account {
   final bool? touch;
   final OathType? oathType;
   final HashAlgorithm? hashAlgorithm;
+
   // final PeriodValues? periodValues;
   // final bool? digits;
 
@@ -110,23 +76,16 @@ extension OathFunctions on WidgetTester {
       await grantCameraPermissions(this);
     }
 
-    /// TODO: reset so this takes input and not overrides with random
-    /// This comes from trying to remove flakiness in the tests.
-    ///
     var issuerText = find.byKey(keys.issuerField).hitTestable();
     await tap(issuerText);
-    // await enterText(issuerText, generateRandomIssuer());
     await enterText(issuerText, a.issuer ?? '');
     await shortWait();
     var nameText = find.byKey(keys.nameField).hitTestable();
     await tap(nameText);
-    // await enterText(nameText, generateRandomName());
     await enterText(nameText, a.name);
     await shortWait();
     var secretText = find.byKey(keys.secretField).hitTestable();
     await tap(secretText);
-    // await generateRandomSecret();
-    // await enterText(issuerText, generateRandomSecret());
     await enterText(secretText, a.secret);
     await shortWait();
     if (a.touch != null && a.touch == true) {
@@ -170,15 +129,10 @@ extension OathFunctions on WidgetTester {
     await shortWait();
     await tap(find.byKey(keys.saveButton));
 
-    /// TODO:
     /// the following pump is because of NEO keys
     await pump(const Duration(seconds: 1));
 
-    /// TODO:
-    /// this verification fails and should be redone:
-    /// "The test failed because the expected value was null, but the actual value was not null"
     accountView = await findAccount(a);
-    //expect(accountView, isNotNull);
     if (accountView != null) {
       testLog(quiet, 'Added account $a');
     }
@@ -263,8 +217,6 @@ extension OathFunctions on WidgetTester {
     await tap(deleteIconButton);
     await longWait();
 
-    /// TODO check dialog shows correct information about account
-
     /// click the delete Button in the delete dialog
     var deleteButton = find.byKey(keys.deleteButton).hitTestable();
     expect(deleteButton, findsOneWidget);
@@ -296,7 +248,6 @@ extension OathFunctions on WidgetTester {
     var renameIconButton = find.byIcon(Icons.edit_outlined).hitTestable();
 
     /// only newer FW supports renaming
-    /// TODO verify this is correct for the FW of the YubiKey
     if (renameIconButton.evaluate().isEmpty) {
       /// close the dialog and return
       testLog(false, 'This YubiKey does not support account renaming');
@@ -323,9 +274,6 @@ extension OathFunctions on WidgetTester {
     await tap(saveButton);
     await longWait();
 
-    /// now the account dialog is shown
-    /// TODO verify it shows correct issuer and name
-
     /// close the account dialog by tapping the close button
     var closeButton = find.byKey(app_keys.closeButton).hitTestable();
     // Wait for toast to clear
@@ -379,7 +327,6 @@ extension OathFunctions on WidgetTester {
 
     await tap(find.byKey(keys.savePasswordButton));
 
-    /// TODO:
     /// the following pause is because of NEO keys
     await pump(const Duration(seconds: 1));
 
@@ -420,7 +367,6 @@ extension OathFunctions on WidgetTester {
     var unlockButton = find.byKey(keys.unlockButton);
     await tap(unlockButton);
 
-    /// TODO:
     /// the following pump is because of NEO keys
     await pump(const Duration(seconds: 1));
 
@@ -438,7 +384,6 @@ extension OathFunctions on WidgetTester {
     await shortWait();
     await tap(find.byKey(keys.removePasswordButton));
 
-    /// TODO:
     /// the following pump is because of NEO keys
     await pump(const Duration(seconds: 1));
 

integration_test/utils/test_util.dart

@@ -227,6 +227,13 @@ extension AppWidgetTester on WidgetTester {
     }
     collectedYubiKeyInformation = true;
   }
+
+  bool isTextButtonEnabled(Key buttonKey) {
+    var finder = find.byKey(buttonKey).hitTestable();
+    expect(finder.evaluate().isNotEmpty, true);
+    TextButton button = finder.evaluate().single.widget as TextButton;
+    return button.enabled;
+  }
 }
 
 @isTest

testrunner.sh

@@ -16,36 +16,39 @@ case "$(uname)" in
 		OS="windows";;
 esac
 
+# directory containing the integration tests
+int_directory="integration_test/"
+
 # Measure the start time
 start_time=$(date +%s.%N)
 
 # Run the keyless tests and measure the time
 keyless_start_time=$(date +%s.%N)
-outputKeyless=$(flutter test -d $OS integration_test/keyless_test.dart)
+outputKeyless=$(flutter test -d $OS "$int_directory"keyless_test.dart)
 keyless_end_time=$(date +%s.%N)
 keyless_time=$(echo "$keyless_end_time - $keyless_start_time" | bc)
 
 # Run the management tests and measure the time
 management_start_time=$(date +%s.%N)
-outputManagement=$(flutter test -d $OS integration_test/management_test.dart)
+outputManagement=$(flutter test -d $OS "$int_directory"management_test.dart)
 management_end_time=$(date +%s.%N)
 management_time=$(echo "$management_end_time - $management_start_time" | bc)
 
 # Run the PIV tests and measure the time
 piv_start_time=$(date +%s.%N)
-outputPiv=$(flutter test -d $OS integration_test/piv_test.dart)
+outputPiv=$(flutter test -d $OS "$int_directory"piv_test.dart)
 piv_end_time=$(date +%s.%N)
 piv_time=$(echo "$piv_end_time - $piv_start_time" | bc)
 
 # Run the OATH tests and measure the time
 oath_start_time=$(date +%s.%N)
-outputOath=$(flutter test -d $OS integration_test/oath_test.dart)
+outputOath=$(flutter test -d $OS "$int_directory"oath_test.dart)
 oath_end_time=$(date +%s.%N)
 oath_time=$(echo "$oath_end_time - $oath_start_time" | bc)
 
 # Run the webauthn tests and measure the time
 webauthn_start_time=$(date +%s.%N)
-outputWebauthn=$(flutter test -d $OS integration_test/webauthn_test.dart)
+outputWebauthn=$(flutter test -d $OS "$int_directory"webauthn_test.dart)
 webauthn_end_time=$(date +%s.%N)
 webauthn_time=$(echo "$webauthn_end_time - $webauthn_start_time" | bc)
 
@@ -58,19 +61,36 @@ total_time=$(echo "$end_time - $start_time" | bc)
 # Output the measured times
 if [[ $outputPiv == *"All tests passed"* ]]; then
     echo "All PIV tests passed: $piv_time seconds"
+else
+    echo "PIV tests failed"
+    echo $outputPiv
 fi
+
 if [[ $outputOath == *"All tests passed"* ]]; then
     echo "All OATH tests passed: $oath_time seconds"
+else
+    echo "OATH tests failed"
+    echo $outputOath
 fi
+
 if [[ $outputWebauthn == *"All tests passed"* ]]; then
     echo "All Webauthn tests passed: $webauthn_time seconds"
+else
+    echo "Webauthn tests failed"
+    echo $outputWebauthn
 fi
 if [[ $outputKeyless == *"All tests passed"* ]]; then
     echo "All Keyless tests passed: $keyless_time seconds"
+else
+    echo "Keyless tests failed"
+    echo $outputKeyless
 fi
+
 if [[ $outputManagement == *"All tests passed"* ]]; then
     echo "All Management tests passed: $management_time seconds"
+else
+    echo "Managemet tests failed"
+    echo $outputManagement
 fi
-#echo "PIV test time: $piv_time seconds"
+
-#echo "OATH test time: $oath_time seconds"
 echo "Total time: $total_time seconds"