diff --git a/doc/Development.adoc b/doc/Development.adoc index 7d56a93f..1c906981 100644 --- a/doc/Development.adoc +++ b/doc/Development.adoc @@ -90,7 +90,7 @@ These do not require a YubiKey, and are relatively quick to run. The integration tests are slower but cover more end-to-end functionality. The require an attached YubiKey to run, and will make modifications to the data stored on that YubiKey. For instructions on running these tests, see -link:Integration_Tests.adoc[these instructions]. +link:../integration_test/testdoc.adoc[these instructions]. === Packaging for MacOS diff --git a/integration_test/keyless_test.dart b/integration_test/keyless_test.dart index 5a9e480a..ed4a25d9 100644 --- a/integration_test/keyless_test.dart +++ b/integration_test/keyless_test.dart @@ -22,7 +22,6 @@ import 'package:yubico_authenticator/app/views/keys.dart'; import 'utils/keyless_test_util.dart'; import 'utils/test_util.dart'; -/// TODO: These need to be able to run keyless to run in CI. void main() { var binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized(); binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive; diff --git a/integration_test/oath_test.dart b/integration_test/oath_test.dart index 65afea5c..84bb02ea 100644 --- a/integration_test/oath_test.dart +++ b/integration_test/oath_test.dart @@ -78,7 +78,6 @@ void main() { await tester.tap(oathDrawerButton); await tester.longWait(); - /// TODO change back to 32 after flakiness eval for (var i = 0; i < 32; i += 1) { // just now merely 32 accounts var testAccount = Account( @@ -104,39 +103,6 @@ void main() { // appTest('Create weird character-accounts and check byte count', // (WidgetTester tester) async {}); group('TOTP account tests', () { - // appTest('Create regular TOTP account', (WidgetTester tester) async { - // // account with issuer field - // // var issuer = generateRandomIssuer(); - // // var name = generateRandomName(); - // // var secret = 'abcdabcd'; - // var testAccount = const Account( - // issuer: 'IssuerForTests', - // name: 'NameForTests', - // secret: 'abcdabcd', - // ); - // var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable(); - // await tester.tap(oathDrawerButton); - // await tester.longWait(); - // - // await tester.addAccount(testAccount); - // await tester.longWait(); - // - // // TODO: Verify account exists - // // TODO: Change testAccount - // await tester.deleteAccount(testAccount); - // }); - // - // appTest('Create issuer-less TOTP account', (WidgetTester tester) async { - // // account without issuer field - // var testAccount = const Account( - // name: 'NoIssuerName', - // secret: 'bbbbbbbbbbbbbbbb', - // ); - // await tester.deleteAccount(testAccount); - // - // /// TODO: change issuer functionality in oath_test_util - // await tester.addAccount(testAccount); - // }); appTest('TOTP: sha-1', (WidgetTester tester) async { var oathDrawerButton = find.byKey(oathAppDrawer).hitTestable(); await tester.tap(oathDrawerButton); diff --git a/integration_test/piv_test.dart b/integration_test/piv_test.dart index 12b0c4a3..a0f1d8f0 100644 --- a/integration_test/piv_test.dart +++ b/integration_test/piv_test.dart @@ -20,7 +20,6 @@ import 'package:integration_test/integration_test.dart'; import 'package:yubico_authenticator/app/views/keys.dart'; import 'package:yubico_authenticator/core/state.dart'; import 'package:yubico_authenticator/piv/keys.dart'; -// import 'package:yubico_authenticator/widgets/tooltip_if_truncated.dart'; import 'utils/piv_test_util.dart'; import 'utils/test_util.dart'; @@ -32,9 +31,6 @@ void main() { group('PIV Settings', skip: isAndroid, () { const factoryPin = '123456'; const factoryPuk = '12345678'; - // TODO: use or remove factoryManagementKey - // const factoryManagemenKey = - // '010203040506070801020304050607080102030405060708'; appTest('Reset PIV (settings-init)', (WidgetTester tester) async { await tester.resetPiv(); await tester.shortWait(); @@ -48,7 +44,7 @@ void main() { await tester.shortWait(); await tester.tap(find.byKey(actionsIconButtonKey).hitTestable()); - /// TODO: This expect needs to verify that Pin underline is 'Blocked' + /// TODO: This expect needs to verify that Pin subtitle is 'Blocked' /// expect(find.byKey(managePinAction), find.byTooltip('Blocked')); await tester.shortWait(); await tester.tap(find.byKey(managePinAction).hitTestable()); @@ -177,9 +173,7 @@ void main() { await tester.enterText( find.byKey(newPinPukField).hitTestable(), shortmanagementkey); await tester.longWait(); - await tester.tap(find.byKey(saveButton).hitTestable()); - await tester.longWait(); - // TODO: verify state + expect(tester.isTextButtonEnabled(saveButton), false); }); appTest('Change managementkey key', (WidgetTester tester) async { await tester.configurePiv(); @@ -260,7 +254,6 @@ void main() { await tester.resetPiv(); }); - /// TODO: The rest of management key settings, when input fields are fixed appTest('Reset PIV (settings-exit)', (WidgetTester tester) async { await tester.resetPiv(); await tester.shortWait(); @@ -367,16 +360,16 @@ void main() { await tester.longWait(); // 9 Verify Subject, verify Date // TODO: this seems not to work! -/* expect(find.byWidgetPredicate((widget) { - if (widget is TooltipIfTruncated) { - final TooltipIfTruncated textWidget = widget; - if (textWidget.key == certInfoSubjectKey && - textWidget.text == 'CN=foobar') { - return true; - } - } - return false; - }), findsOneWidget);*/ + // expect(find.byWidgetPredicate((widget) { + // if (widget is TooltipIfTruncated) { + // final TooltipIfTruncated textWidget = widget; + // if (textWidget.key == certInfoSubjectKey && + // textWidget.text == 'CN=Generate9c') { + // return true; + // } + // } + // return false; + // }), findsOneWidget); await tester.longWait(); // 10. Delete Certificate @@ -491,58 +484,14 @@ void main() { await tester.tap(find.byKey(deleteButton).hitTestable()); await tester.longWait(); }); - // appTest('Import outdated Key+Certificate from file', - // (WidgetTester tester) async { - // /// TODO fileload needs to be handled - // // 1. open PIV view - // var pivDrawerButton = find.byKey(pivAppDrawer).hitTestable(); - // await tester.tap(pivDrawerButton); - // await tester.longWait(); - // // 2. click meatball menu for 9c - // await tester.tap(find.byKey(meatballButton9c).hitTestable()); - // await tester.longWait(); - // // 3. click import - // await tester.tap(find.byKey(importAction).hitTestable()); - // await tester.longWait(); - // // 4. pick key: outdated_key.pem and "Choose" - // // 5. TODO: tap close - // // 6. Verify slot 9c "Key without certificate loaded" - // // 7. click meatball menu for 9c - // await tester.tap(find.byKey(meatballButton9c).hitTestable()); - // await tester.longWait(); - // // 8. click import - // await tester.tap(find.byKey(importAction).hitTestable()); - // await tester.longWait(); - // // 9. pick key: outdated_cert.pem and "Choose" - // // 10. Tap "Import" on 'Import File Dialogue' - // // Verify Certificate - // }); - // appTest('Import neverexpire Key+Certificate from file', - // (WidgetTester tester) async { - // /// TODO fileload needs to be handled - // // // 1. open PIV view - // // var pivDrawerButton = find.byKey(pivAppDrawer).hitTestable(); - // // await tester.tap(pivDrawerButton); - // // await tester.longWait(); - // // // 2. click meatball menu for 9d - // // await tester.tap(find.byKey(meatballButton9d).hitTestable()); - // // await tester.longWait(); - // // // 3. click import - // // await tester.tap(find.byKey(importAction).hitTestable()); - // // await tester.longWait(); - // // // 4. pick key: neverexpire_key.pem and "Choose" - // // // 5. TODO: tap close - // // // 6. Verify slot 9c "Key without certificate loaded" - // // // 7. click meatball menu for 9d - // // await tester.tap(find.byKey(meatballButton9d).hitTestable()); - // // await tester.longWait(); - // // // 8. click import - // // await tester.tap(find.byKey(importAction).hitTestable()); - // // await tester.longWait(); - // // // 9. pick key: neverexpire_cert.pem and "Choose" - // // // 10. Tap "Import" on 'Import File Dialogue' - // // // Verify Certificate - // }); + appTest('Import outdated Key+Certificate from file', + (WidgetTester tester) async {}); + + /// TODO fileload needs to be handled + appTest('Import neverexpire Key+Certificate from file', + (WidgetTester tester) async { + /// TODO fileload needs to be handled + }); appTest('Generate a CSR', (WidgetTester tester) async { // 1. open PIV view var pivDrawerButton = find.byKey(pivAppDrawer).hitTestable(); diff --git a/integration_test/testdoc.adoc b/integration_test/testdoc.adoc index cac02b7f..e27178be 100644 --- a/integration_test/testdoc.adoc +++ b/integration_test/testdoc.adoc @@ -1,8 +1,10 @@ == Testing Yubico Authenticator -We have set up testing of the Yubico Authenticator flutter as a part of one of the following groups. +Verifying the quality of Yubico Authenticator is made through multiple levels of tests, loosely +defined in the following groups. * Automatic tests in CI pipeline +* Unit tests * Automatic tests requiring a Yubikey * Manual tests @@ -12,7 +14,7 @@ as they depend on other technology which are a bit to expensive to automate (cam the flutter test framework. To run the tests you need to specify the serial number of a Yubikey which the tests are able to run -on (OBS this key will be reset and by running the tests). +on (OBS this key will be reset by running the tests, so don't use one of your personal keys). === Desktop Running the tests for the CI environment: @@ -21,9 +23,18 @@ Running the tests for the CI environment: Running all the tests for desktop (requires a Yubikey): - $ flutter test integration_test + $ ./testrunner.sh -We will have this handled through a shell interface, more info to come. +Running all the tests for desktop (requires a Yubikey): + + $ flutter test + +The testrunner interface is work in progress, but it deals with a problem of the flutter +integration_test framework, i.e. that it fails to initiate multiple UI-tests. + +Manual test scripts will be moved here so all quality related information is in the same place. === Android -TBD + +Testing of GUI of the Android client is also a work in progress as tests work differently on the +mobile platform. diff --git a/integration_test/utils/keyless_test_util.dart b/integration_test/utils/keyless_test_util.dart index 6238cfb7..960b42ba 100644 --- a/integration_test/utils/keyless_test_util.dart +++ b/integration_test/utils/keyless_test_util.dart @@ -29,7 +29,6 @@ String? yubiKeyFirmware; String? yubiKeySerialNumber; bool collectedYubiKeyInformation = false; -/// TODO: clean up this monster of appTestKeyLess extension AppWidgetTester on WidgetTester { /// waits up to [timeOutSec] seconds evaluating whether [Finder] f is /// visible diff --git a/integration_test/utils/oath_test_util.dart b/integration_test/utils/oath_test_util.dart index e732bdf2..caec236f 100644 --- a/integration_test/utils/oath_test_util.dart +++ b/integration_test/utils/oath_test_util.dart @@ -14,9 +14,6 @@ * limitations under the License. */ -import 'dart:convert'; -import 'dart:math'; - import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:yubico_authenticator/app/views/keys.dart' as app_keys; @@ -29,38 +26,6 @@ import 'package:yubico_authenticator/oath/views/account_view.dart'; import '../utils/test_util.dart'; import 'android/util.dart'; -/// THESE SHOULD PROBABLY BE REMOVOVED: -/// -String randomPadded() { - return randomNum(999).toString().padLeft(3, '0'); -} - -randomNum(int i) {} - -String generateRandomIssuer() { - final random = Random.secure(); - return 'issuer_${base64Encode(List.generate(4, (_) => random.nextInt(256)))}'; - // return 'i${randomPadded()}'; -} - -String generateRandomName() { - final random = Random.secure(); - return 'name_${base64Encode(List.generate(4, (_) => random.nextInt(256)))}'; - //return 'n${randomPadded()}'; -} - -String generateRandomSecret() { - final random = Random.secure(); - return base64Encode(List.generate(8, (_) => random.nextInt(256))); -} - -String staticSecret() { - return 'abba'; -} - -/// -/// THESE SHOULD PROBABLY BE REMOVOVED - class Account { final String? issuer; final String name; @@ -68,6 +33,7 @@ class Account { final bool? touch; final OathType? oathType; final HashAlgorithm? hashAlgorithm; + // final PeriodValues? periodValues; // final bool? digits; @@ -110,23 +76,16 @@ extension OathFunctions on WidgetTester { await grantCameraPermissions(this); } - /// TODO: reset so this takes input and not overrides with random - /// This comes from trying to remove flakiness in the tests. - /// var issuerText = find.byKey(keys.issuerField).hitTestable(); await tap(issuerText); - // await enterText(issuerText, generateRandomIssuer()); await enterText(issuerText, a.issuer ?? ''); await shortWait(); var nameText = find.byKey(keys.nameField).hitTestable(); await tap(nameText); - // await enterText(nameText, generateRandomName()); await enterText(nameText, a.name); await shortWait(); var secretText = find.byKey(keys.secretField).hitTestable(); await tap(secretText); - // await generateRandomSecret(); - // await enterText(issuerText, generateRandomSecret()); await enterText(secretText, a.secret); await shortWait(); if (a.touch != null && a.touch == true) { @@ -170,15 +129,10 @@ extension OathFunctions on WidgetTester { await shortWait(); await tap(find.byKey(keys.saveButton)); - /// TODO: /// the following pump is because of NEO keys await pump(const Duration(seconds: 1)); - /// TODO: - /// this verification fails and should be redone: - /// "The test failed because the expected value was null, but the actual value was not null" accountView = await findAccount(a); - //expect(accountView, isNotNull); if (accountView != null) { testLog(quiet, 'Added account $a'); } @@ -263,8 +217,6 @@ extension OathFunctions on WidgetTester { await tap(deleteIconButton); await longWait(); - /// TODO check dialog shows correct information about account - /// click the delete Button in the delete dialog var deleteButton = find.byKey(keys.deleteButton).hitTestable(); expect(deleteButton, findsOneWidget); @@ -296,7 +248,6 @@ extension OathFunctions on WidgetTester { var renameIconButton = find.byIcon(Icons.edit_outlined).hitTestable(); /// only newer FW supports renaming - /// TODO verify this is correct for the FW of the YubiKey if (renameIconButton.evaluate().isEmpty) { /// close the dialog and return testLog(false, 'This YubiKey does not support account renaming'); @@ -323,9 +274,6 @@ extension OathFunctions on WidgetTester { await tap(saveButton); await longWait(); - /// now the account dialog is shown - /// TODO verify it shows correct issuer and name - /// close the account dialog by tapping the close button var closeButton = find.byKey(app_keys.closeButton).hitTestable(); // Wait for toast to clear @@ -379,7 +327,6 @@ extension OathFunctions on WidgetTester { await tap(find.byKey(keys.savePasswordButton)); - /// TODO: /// the following pause is because of NEO keys await pump(const Duration(seconds: 1)); @@ -420,7 +367,6 @@ extension OathFunctions on WidgetTester { var unlockButton = find.byKey(keys.unlockButton); await tap(unlockButton); - /// TODO: /// the following pump is because of NEO keys await pump(const Duration(seconds: 1)); @@ -438,7 +384,6 @@ extension OathFunctions on WidgetTester { await shortWait(); await tap(find.byKey(keys.removePasswordButton)); - /// TODO: /// the following pump is because of NEO keys await pump(const Duration(seconds: 1)); diff --git a/integration_test/utils/test_util.dart b/integration_test/utils/test_util.dart index aa595f03..5442382d 100644 --- a/integration_test/utils/test_util.dart +++ b/integration_test/utils/test_util.dart @@ -227,6 +227,13 @@ extension AppWidgetTester on WidgetTester { } collectedYubiKeyInformation = true; } + + bool isTextButtonEnabled(Key buttonKey) { + var finder = find.byKey(buttonKey).hitTestable(); + expect(finder.evaluate().isNotEmpty, true); + TextButton button = finder.evaluate().single.widget as TextButton; + return button.enabled; + } } @isTest diff --git a/integration_test/testrunner.sh b/testrunner.sh similarity index 73% rename from integration_test/testrunner.sh rename to testrunner.sh index 39e66eff..1f0d66eb 100755 --- a/integration_test/testrunner.sh +++ b/testrunner.sh @@ -16,36 +16,39 @@ case "$(uname)" in OS="windows";; esac +# directory containing the integration tests +int_directory="integration_test/" + # Measure the start time start_time=$(date +%s.%N) # Run the keyless tests and measure the time keyless_start_time=$(date +%s.%N) -outputKeyless=$(flutter test -d $OS integration_test/keyless_test.dart) +outputKeyless=$(flutter test -d $OS "$int_directory"keyless_test.dart) keyless_end_time=$(date +%s.%N) keyless_time=$(echo "$keyless_end_time - $keyless_start_time" | bc) # Run the management tests and measure the time management_start_time=$(date +%s.%N) -outputManagement=$(flutter test -d $OS integration_test/management_test.dart) +outputManagement=$(flutter test -d $OS "$int_directory"management_test.dart) management_end_time=$(date +%s.%N) management_time=$(echo "$management_end_time - $management_start_time" | bc) # Run the PIV tests and measure the time piv_start_time=$(date +%s.%N) -outputPiv=$(flutter test -d $OS integration_test/piv_test.dart) +outputPiv=$(flutter test -d $OS "$int_directory"piv_test.dart) piv_end_time=$(date +%s.%N) piv_time=$(echo "$piv_end_time - $piv_start_time" | bc) # Run the OATH tests and measure the time oath_start_time=$(date +%s.%N) -outputOath=$(flutter test -d $OS integration_test/oath_test.dart) +outputOath=$(flutter test -d $OS "$int_directory"oath_test.dart) oath_end_time=$(date +%s.%N) oath_time=$(echo "$oath_end_time - $oath_start_time" | bc) # Run the webauthn tests and measure the time webauthn_start_time=$(date +%s.%N) -outputWebauthn=$(flutter test -d $OS integration_test/webauthn_test.dart) +outputWebauthn=$(flutter test -d $OS "$int_directory"webauthn_test.dart) webauthn_end_time=$(date +%s.%N) webauthn_time=$(echo "$webauthn_end_time - $webauthn_start_time" | bc) @@ -58,19 +61,36 @@ total_time=$(echo "$end_time - $start_time" | bc) # Output the measured times if [[ $outputPiv == *"All tests passed"* ]]; then echo "All PIV tests passed: $piv_time seconds" +else + echo "PIV tests failed" + echo $outputPiv fi + if [[ $outputOath == *"All tests passed"* ]]; then echo "All OATH tests passed: $oath_time seconds" +else + echo "OATH tests failed" + echo $outputOath fi + if [[ $outputWebauthn == *"All tests passed"* ]]; then echo "All Webauthn tests passed: $webauthn_time seconds" +else + echo "Webauthn tests failed" + echo $outputWebauthn fi if [[ $outputKeyless == *"All tests passed"* ]]; then echo "All Keyless tests passed: $keyless_time seconds" +else + echo "Keyless tests failed" + echo $outputKeyless fi + if [[ $outputManagement == *"All tests passed"* ]]; then echo "All Management tests passed: $management_time seconds" +else + echo "Managemet tests failed" + echo $outputManagement fi -#echo "PIV test time: $piv_time seconds" -#echo "OATH test time: $oath_time seconds" + echo "Total time: $total_time seconds" \ No newline at end of file