From 9bf11d9fd20576165a5c8c4ece68944abeea9dba Mon Sep 17 00:00:00 2001 From: hedger Date: Thu, 6 Oct 2022 17:55:57 +0400 Subject: [PATCH 1/2] [FL-2859,2838] fbt: improvements for FAPs (#1813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fbt: assets builder for apps WIP * fbt: automatically building private fap assets * docs: details on how to use image assets * fbt: renamed fap_assets -> fap_icons * fbt: support for fap_extbuild field * docs: info on fap_extbuild * fbt: added --proxy-env parame ter * fbt: made firmware_cdb & updater_cdb targets always available * fbt: renamed fap_icons -> fap_icon_assets * fbt: deprecated firmware_* target names for faps; new alias is "fap_APPID" * fbt: changed intermediate file locations for external apps * fbt: support for fap_private_libs; docs: updates * restored mbedtls as global lib * scripts: lint.py: skip "lib" subfolder * fbt: Sanity checks for building advanced faps as part of fw * docs: info on fap_private_libs; fbt: optimized *.fam indexing * fbt: cleanup; samples: added sample_icons app * fbt: moved example app to applications/examples * linter fix * docs: readme fixes * added applications/examples/application.fam stub * docs: more info on private libs Co-authored-by: あく --- applications/examples/application.fam | 5 + .../examples/example_images/application.fam | 10 ++ .../examples/example_images/example_images.c | 79 +++++++++ .../example_images/images/dolphin_71x25.png | Bin 0 -> 1188 bytes applications/plugins/picopass/application.fam | 9 +- .../{ => lib}/loclass/optimized_cipher.c | 0 .../{ => lib}/loclass/optimized_cipher.h | 0 .../{ => lib}/loclass/optimized_cipherutils.c | 0 .../{ => lib}/loclass/optimized_cipherutils.h | 0 .../{ => lib}/loclass/optimized_elite.c | 0 .../{ => lib}/loclass/optimized_elite.h | 0 .../{ => lib}/loclass/optimized_ikeys.c | 0 .../{ => lib}/loclass/optimized_ikeys.h | 0 .../plugins/picopass/picopass_device.h | 4 +- assets/SConscript | 19 +-- documentation/AppManifests.md | 57 ++++++- documentation/AppsOnSDCard.md | 17 +- documentation/fbt.md | 11 +- firmware.scons | 35 ++-- scripts/assets.py | 19 ++- scripts/lint.py | 4 + site_scons/commandline.scons | 9 ++ site_scons/environ.scons | 10 +- site_scons/extapps.scons | 26 ++- site_scons/fbt/appmanifest.py | 25 +++ site_scons/site_tools/fbt_assets.py | 30 +++- site_scons/site_tools/fbt_extapps.py | 150 ++++++++++++++---- 27 files changed, 438 insertions(+), 81 deletions(-) create mode 100644 applications/examples/application.fam create mode 100644 applications/examples/example_images/application.fam create mode 100644 applications/examples/example_images/example_images.c create mode 100644 applications/examples/example_images/images/dolphin_71x25.png rename applications/plugins/picopass/{ => lib}/loclass/optimized_cipher.c (100%) rename applications/plugins/picopass/{ => lib}/loclass/optimized_cipher.h (100%) rename applications/plugins/picopass/{ => lib}/loclass/optimized_cipherutils.c (100%) rename applications/plugins/picopass/{ => lib}/loclass/optimized_cipherutils.h (100%) rename applications/plugins/picopass/{ => lib}/loclass/optimized_elite.c (100%) rename applications/plugins/picopass/{ => lib}/loclass/optimized_elite.h (100%) rename applications/plugins/picopass/{ => lib}/loclass/optimized_ikeys.c (100%) rename applications/plugins/picopass/{ => lib}/loclass/optimized_ikeys.h (100%) diff --git a/applications/examples/application.fam b/applications/examples/application.fam new file mode 100644 index 000000000..16d240ccf --- /dev/null +++ b/applications/examples/application.fam @@ -0,0 +1,5 @@ +App( + appid="sample_apps", + name="Sample apps bundle", + apptype=FlipperAppType.METAPACKAGE, +) diff --git a/applications/examples/example_images/application.fam b/applications/examples/example_images/application.fam new file mode 100644 index 000000000..9a5f8e030 --- /dev/null +++ b/applications/examples/example_images/application.fam @@ -0,0 +1,10 @@ +App( + appid="example_images", + name="Example: Images", + apptype=FlipperAppType.EXTERNAL, + entry_point="example_images_main", + requires=["gui"], + stack_size=1 * 1024, + fap_category="Examples", + fap_icon_assets="images", +) diff --git a/applications/examples/example_images/example_images.c b/applications/examples/example_images/example_images.c new file mode 100644 index 000000000..48fa5e77e --- /dev/null +++ b/applications/examples/example_images/example_images.c @@ -0,0 +1,79 @@ +#include +#include + +#include +#include + +#include "example_images_icons.h" + +typedef struct { + uint8_t x, y; +} ImagePosition; + +static ImagePosition image_position = {.x = 0, .y = 0}; + +// Screen is 128x64 px +static void app_draw_callback(Canvas* canvas, void* ctx) { + UNUSED(ctx); + + canvas_clear(canvas); + canvas_draw_icon(canvas, image_position.x % 128, image_position.y % 64, &I_dolphin_71x25); +} + +static void app_input_callback(InputEvent* input_event, void* ctx) { + furi_assert(ctx); + + FuriMessageQueue* event_queue = ctx; + furi_message_queue_put(event_queue, input_event, FuriWaitForever); +} + +int32_t example_images_main(void* p) { + UNUSED(p); + FuriMessageQueue* event_queue = furi_message_queue_alloc(8, sizeof(InputEvent)); + + // Configure view port + ViewPort* view_port = view_port_alloc(); + view_port_draw_callback_set(view_port, app_draw_callback, view_port); + view_port_input_callback_set(view_port, app_input_callback, event_queue); + + // Register view port in GUI + Gui* gui = furi_record_open(RECORD_GUI); + gui_add_view_port(gui, view_port, GuiLayerFullscreen); + + InputEvent event; + + bool running = true; + while(running) { + if(furi_message_queue_get(event_queue, &event, 100) == FuriStatusOk) { + if((event.type == InputTypePress) || (event.type == InputTypeRepeat)) { + switch(event.key) { + case InputKeyLeft: + image_position.x -= 2; + break; + case InputKeyRight: + image_position.x += 2; + break; + case InputKeyUp: + image_position.y -= 2; + break; + case InputKeyDown: + image_position.y += 2; + break; + default: + running = false; + break; + } + } + } + view_port_update(view_port); + } + + view_port_enabled_set(view_port, false); + gui_remove_view_port(gui, view_port); + view_port_free(view_port); + furi_message_queue_free(event_queue); + + furi_record_close(RECORD_GUI); + + return 0; +} diff --git a/applications/examples/example_images/images/dolphin_71x25.png b/applications/examples/example_images/images/dolphin_71x25.png new file mode 100644 index 0000000000000000000000000000000000000000..6b3f8aa59b91c644a942bbe12f16c93f4c2d4619 GIT binary patch literal 1188 zcmaJ>TWAzl7@kEf7@I;~nks=zCO(*wx$f*`#%yF}XLqwDyN&A>qX;^elkAYpoN;E7 zT_ad+1+A!okV0Py4WTxu1Va$ocu7nty*w!DqxewhOYlKo`cO5bXEwXhhv>kWbN(~m z_uv2drZ1mqY}nO+VOV3fM=78^gVxT_7W6*!a_R}%LS7*wW3%^LR*YCPxa}3AQ3{SH>$uMGA5P2T2Jp{6c<6W*XAQqH#%^s2xM9KFZk*3S#GF1*!&>f^% zK@ez$qdAU52+})Y`)Y->z4mn_H8l$Gbk}rz6WVy7R@LB$pCFLS>#V-n3T8~@-t~m;fvub zaw?2&QUafr!$gg1Y?8kkH}Y;SU2Q?+5*@V5TkW&nn$=s>o55hv8Yf7xu#|2!_;DXYyRGERL7p5^$ #include "rfal_picopass.h" -#include "loclass/optimized_ikeys.h" -#include "loclass/optimized_cipher.h" +#include +#include #define PICOPASS_DEV_NAME_MAX_LEN 22 #define PICOPASS_READER_DATA_MAX_SIZE 64 diff --git a/assets/SConscript b/assets/SConscript index a0b3b13ab..e1bf546cc 100644 --- a/assets/SConscript +++ b/assets/SConscript @@ -9,28 +9,13 @@ assetsenv = env.Clone( ) assetsenv.ApplyLibFlags() -if not assetsenv["VERBOSE"]: - assetsenv.SetDefault( - ICONSCOMSTR="\tICONS\t${TARGET}", - PROTOCOMSTR="\tPROTO\t${SOURCE}", - DOLPHINCOMSTR="\tDOLPHIN\t${DOLPHIN_RES_TYPE}", - RESMANIFESTCOMSTR="\tMANIFEST\t${TARGET}", - PBVERCOMSTR="\tPBVER\t${TARGET}", - ) - -# Gathering icons sources -icons_src = assetsenv.GlobRecursive("*.png", "icons") -icons_src += assetsenv.GlobRecursive("frame_rate", "icons") - -icons = assetsenv.IconBuilder( - assetsenv.Dir("compiled"), ICON_SRC_DIR=assetsenv.Dir("#/assets/icons") +icons = assetsenv.CompileIcons( + assetsenv.Dir("compiled"), assetsenv.Dir("#/assets/icons") ) -assetsenv.Depends(icons, icons_src) assetsenv.Alias("icons", icons) # Protobuf .proto -> .c + .h - proto_src = assetsenv.Glob("protobuf/*.proto", source=True) proto_options = assetsenv.Glob("protobuf/*.options", source=True) proto = assetsenv.ProtoBuilder(assetsenv.Dir("compiled"), proto_src) diff --git a/documentation/AppManifests.md b/documentation/AppManifests.md index 0a4f5e9b7..14c0ae3ac 100644 --- a/documentation/AppManifests.md +++ b/documentation/AppManifests.md @@ -41,9 +41,12 @@ Only 2 parameters are mandatory: ***appid*** and ***apptype***, others are optio * **order**: Order of an application within its group when sorting entries in it. The lower the order is, the closer to the start of the list the item is placed. *Used for ordering startup hooks and menu entries.* * **sdk_headers**: List of C header files from this app's code to include in API definitions for external applications. + +#### Parameters for external applications + The following parameters are used only for [FAPs](./AppsOnSDCard.md): -* **sources**: list of strings, file name masks, used for gathering sources within app folder. Default value of `["*.c*"]` includes C and CPP source files. +* **sources**: list of strings, file name masks, used for gathering sources within app folder. Default value of `["*.c*"]` includes C and C++ source files. Application cannot use `"lib"` folder for their own source code, as it is reserved for **fap_private_libs**. * **fap_version**: tuple, 2 numbers in form of (x,y): application version to be embedded within .fap file. Default value is (0,1), meanig version "0.1". * **fap_icon**: name of a .png file, 1-bit color depth, 10x10px, to be embedded within .fap file. * **fap_libs**: list of extra libraries to link application against. Provides access to extra functions that are not exported as a part of main firmware at expense of increased .fap file size and RAM consumption. @@ -51,6 +54,58 @@ The following parameters are used only for [FAPs](./AppsOnSDCard.md): * **fap_description**: string, may be empty. Short application description. * **fap_author**: string, may be empty. Application's author. * **fap_weburl**: string, may be empty. Application's homepage. +* **fap_icon_assets**: string. If present, defines a folder name to be used for gathering image assets for this application. These images will be preprocessed and built alongside the application. See [FAP assets](./AppsOnSDCard.md#fap-assets) for details. +* **fap_extbuild**: provides support for parts of application sources to be build by external tools. Contains a list of `ExtFile(path="file name", command="shell command")` definitions. **`fbt`** will run the specified command for each file in the list. +Note that commands are executed at the firmware root folder's root, and all intermediate files must be placed in a application's temporary build folder. For that, you can use pattern expansion by **`fbt`**: `${FAP_WORK_DIR}` will be replaced with the path to the application's temporary build folder, and `${FAP_SRC_DIR}` will be replaced with the path to the application's source folder. You can also use other variables defined internally by **`fbt`**. + +Example for building an app from Rust sources: + +```python + sources=["target/thumbv7em-none-eabihf/release/libhello_rust.a"], + fap_extbuild=( + ExtFile( + path="${FAP_WORK_DIR}/target/thumbv7em-none-eabihf/release/libhello_rust.a", + command="cargo build --release --verbose --target thumbv7em-none-eabihf --target-dir ${FAP_WORK_DIR}/target --manifest-path ${FAP_SRC_DIR}/Cargo.toml", + ), + ), +``` + +* **fap_private_libs**: list of additional libraries that are distributed as sources alongside the application. These libraries will be built as a part of the application build process. +Library sources must be placed in a subfolder of "`lib`" folder within the application's source folder. +Each library is defined as a call to `Lib()` function, accepting the following parameters: + + - **name**: name of library's folder. Required. + - **fap_include_paths**: list of library's relative paths to add to parent fap's include path list. Default value is `["."]` meaning library's source root. + - **sources**: list of filename masks to be used for gathering include files for this library. Default value is `["*.c*"]`. + - **cflags**: list of additional compiler flags to be used for building this library. Default value is `[]`. + - **cdefines**: list of additional preprocessor definitions to be used for building this library. Default value is `[]`. + - **cincludes**: list of additional include paths to be used for building this library. Can be used for providing external search paths for this library's code - for configuration headers. Default value is `[]`. + +Example for building an app with a private library: + +```python + fap_private_libs=[ + Lib( + name="mbedtls", + fap_include_paths=["include"], + sources=[ + "library/des.c", + "library/sha1.c", + "library/platform_util.c", + ], + cdefines=["MBEDTLS_ERROR_C"], + ), + Lib( + name="loclass", + cflags=["-Wno-error"], + ), + ], +``` + +For that snippet, **`fbt`** will build 2 libraries: one from sources in `lib/mbedtls` folder, and another from sources in `lib/loclass` folder. For `mbedtls` library, **`fbt`** will add `lib/mbedtls/include` to the list of include paths for the application and compile only the files specified in `sources` list. Additionally, **`fbt`** will enable `MBEDTLS_ERROR_C` preprocessor definition for `mbedtls` sources. +For `loclass` library, **`fbt`** will add `lib/loclass` to the list of include paths for the application and build all sources in that folder. Also **`fbt`** will disable treating compiler warnings as errors for `loclass` library specifically - that can be useful when compiling large 3rd-party codebases. + +Both libraries will be linked into the application. ## .fam file contents diff --git a/documentation/AppsOnSDCard.md b/documentation/AppsOnSDCard.md index 525821530..552be13e4 100644 --- a/documentation/AppsOnSDCard.md +++ b/documentation/AppsOnSDCard.md @@ -2,7 +2,7 @@ [fbt](./fbt.md) has support for building applications as FAP files. FAP are essentially .elf executables with extra metadata and resources bundled in. -FAPs are built with `firmware_extapps` (or `plugin_dist`) **`fbt`** targets. +FAPs are built with `faps` **`fbt`** target. They can also be deployed to `dist` folder with `plugin_dist` **`fbt`** target. FAPs do not depend on being run on a specific firmware version. Compatibility is determined by the FAP's metadata, which includes the required [API version](#api-versioning). @@ -18,6 +18,17 @@ To build your application as a FAP, just create a folder with your app's source * To build all FAPs, run `./fbt plugin_dist`. +## FAP assets + +FAPs can include static and animated images as private assets. They will be automatically compiled alongside application sources and can be referenced the same way as assets from the main firmware. + +To use that feature, put your images in a subfolder inside your application's folder, then reference that folder in your application's manifest in `fap_icon_assets` field. See [Application Manifests](./AppManifests.md#application-definition) for more details. + +To use these assets in your application, put `#include "{APPID}_icons.h"` in your application's source code, where `{APPID}` is the `appid` value field from your application's manifest. Then you can use all icons from your application's assets the same way as if they were a part of `assets_icons.h` of the main firmware. + +Images and animated icons must follow the same [naming convention](../assets/ReadMe.md#asset-naming-rules) as those from the main firmware. + + ## Debugging FAPs **`fbt`** includes a script for gdb-py to provide debugging support for FAPs, `debug/flipperapps.py`. It is loaded in default debugging configurations by **`fbt`** and stock VSCode configurations. @@ -53,13 +64,13 @@ App loader allocates memory for the application and copies it to RAM, processing Not all parts of firmware are available for external applications. A subset of available functions and variables is defined in "api_symbols.csv" file, which is a part of firmware target definition in `firmware/targets/` directory. -**`fbt`** uses semantic versioning for API versioning. Major version is incremented when there are breaking changes in the API, minor version is incremented when there are new features added. +**`fbt`** uses semantic versioning for API. Major version is incremented when there are breaking changes in the API, minor version is incremented when new features are added. Breaking changes include: - removal of a function or a global variable; - changing the signature of a function. -API versioning is mostly automated by **`fbt`**. When rebuilding the firmware, **`fbt`** checks if there are any changes in the API exposed by headers gathered from `SDK_HEADERS`. If there are, it stops the build, adjusts the API version and asks the user to go through the changes in .csv file. New entries are marked with "`?`" mark, and the user is supposed to change the mark to "`+`" for the entry to be exposed for FAPs, "`-`" for it to be unavailable. +API versioning is mostly automated by **`fbt`**. When rebuilding the firmware, **`fbt`** checks if there are any changes in the API exposed by headers gathered from `SDK_HEADERS`. If so, it stops the build, adjusts the API version and asks the user to go through the changes in .csv file. New entries are marked with "`?`" mark, and the user is supposed to change the mark to "`+`" for the entry to be exposed for FAPs, "`-`" for it to be unavailable. **`fbt`** will not allow building a firmware until all "`?`" entries are changed to "`+`" or "`-`". diff --git a/documentation/fbt.md b/documentation/fbt.md index 090ff78f0..e20d43177 100644 --- a/documentation/fbt.md +++ b/documentation/fbt.md @@ -59,10 +59,11 @@ To run cleanup (think of `make clean`) for specified targets, add `-c` option. ### Firmware targets -- `firmware_extapps` - build all plug-ins as separate .elf files - - `firmware_snake_game`, etc - build single plug-in as .elf by its name - - Check out `--extra-ext-apps` for force adding extra apps to external build - - `firmware_snake_game_list`, etc - generate source + assembler listing for app's .elf +- `faps` - build all external & plugin apps as [.faps](./AppsOnSDCard.md#fap-flipper-application-package). +- **`fbt`** also defines per-app targets. For example, for an app with `appid=snake_game` target names are: + - `fap_snake_game`, etc - build single app as .fap by its application ID. + - Check out [`--extra-ext-apps`](#command-line-parameters) for force adding extra apps to external build + - `fap_snake_game_list`, etc - generate source + assembler listing for app's .fap - `flash`, `firmware_flash` - flash current version to attached device with OpenOCD over ST-Link - `jflash` - flash current version to attached device with JFlash using J-Link probe. JFlash executable must be on your $PATH - `flash_blackmagic` - flash current version to attached device with Blackmagic probe @@ -83,9 +84,9 @@ To run cleanup (think of `make clean`) for specified targets, add `-c` option. ## Command-line parameters - `--options optionfile.py` (default value `fbt_options.py`) - load file with multiple configuration values -- `--with-updater` - enables updater-related targets and dependency tracking. Enabling this option introduces extra startup time costs, so use it when bundling update packages. _Explicily enabling this should no longer be required, **`fbt`** now has specific handling for updater-related targets_ - `--extra-int-apps=app1,app2,appN` - forces listed apps to be built as internal with `firmware` target - `--extra-ext-apps=app1,app2,appN` - forces listed apps to be built as external with `firmware_extapps` target +- `--proxy-env=VAR1,VAR2` - additional environment variables to expose to subprocesses spawned by `fbt`. By default, `fbt` sanitizes execution environment and doesn't forward all inherited environment variables. You can find list of variables that are always forwarded in `environ.scons` file. ## Configuration diff --git a/firmware.scons b/firmware.scons index 0970541e8..dd13b6b3d 100644 --- a/firmware.scons +++ b/firmware.scons @@ -1,5 +1,6 @@ Import("ENV", "fw_build_meta") +from SCons.Errors import UserError import itertools from fbt.util import ( @@ -164,13 +165,25 @@ apps_c = fwenv.ApplicationsC( # Adding dependency on manifest files so apps.c is rebuilt when any manifest is changed for app_dir, _ in env["APPDIRS"]: app_dir_node = env.Dir("#").Dir(app_dir) - fwenv.Depends(apps_c, fwenv.GlobRecursive("*.fam", app_dir_node)) + fwenv.Depends(apps_c, app_dir_node.glob("*/application.fam")) + +# Sanity check - certain external apps are using features that are not available in base firmware +if advanced_faps := list( + filter( + lambda app: app.fap_extbuild or app.fap_private_libs or app.fap_icon_assets, + fwenv["APPBUILD"].get_builtin_apps(), + ) +): + raise UserError( + "An Application that is using fap-specific features cannot be built into base firmware." + f" Offending app(s): {', '.join(app.appid for app in advanced_faps)}" + ) sources = [apps_c] # Gather sources only from app folders in current configuration sources.extend( itertools.chain.from_iterable( - fwenv.GlobRecursive(source_type, appdir.relpath) + fwenv.GlobRecursive(source_type, appdir.relpath, exclude="lib") for appdir, source_type in fwenv["APPBUILD"].get_builtin_app_folders() ) ) @@ -259,18 +272,18 @@ fw_artifacts = fwenv["FW_ARTIFACTS"] = [ fwenv["FW_VERSION_JSON"], ] + +fwcdb = fwenv.CompilationDatabase() +# without filtering, both updater & firmware commands would be generated in same file +fwenv.Replace(COMPILATIONDB_PATH_FILTER=fwenv.subst("*${FW_FLAVOR}*")) +AlwaysBuild(fwcdb) +Precious(fwcdb) +NoClean(fwcdb) +Alias(fwenv["FIRMWARE_BUILD_CFG"] + "_cdb", fwcdb) + # If current configuration was explicitly requested, generate compilation database # and link its directory as build/latest if should_gen_cdb_and_link_dir(fwenv, BUILD_TARGETS): - fwcdb = fwenv.CompilationDatabase() - # without filtering, both updater & firmware commands would be generated - fwenv.Replace(COMPILATIONDB_PATH_FILTER=fwenv.subst("*${FW_FLAVOR}*")) - AlwaysBuild(fwcdb) - Precious(fwcdb) - NoClean(fwcdb) - Alias(fwenv["FIRMWARE_BUILD_CFG"] + "_cdb", fwcdb) - AlwaysBuild(fwenv["FIRMWARE_BUILD_CFG"] + "_cdb", fwcdb) - Alias(fwcdb, "") fw_artifacts.append(fwcdb) # Adding as a phony target, so folder link is updated even if elf didn't change diff --git a/scripts/assets.py b/scripts/assets.py index 9b4ee5b61..75bebcfb4 100755 --- a/scripts/assets.py +++ b/scripts/assets.py @@ -14,7 +14,7 @@ ICONS_TEMPLATE_H_HEADER = """#pragma once """ ICONS_TEMPLATE_H_ICON_NAME = "extern const Icon {name};\n" -ICONS_TEMPLATE_C_HEADER = """#include \"assets_icons.h\" +ICONS_TEMPLATE_C_HEADER = """#include "{assets_filename}.h" #include @@ -33,6 +33,13 @@ class Main(App): ) self.parser_icons.add_argument("input_directory", help="Source directory") self.parser_icons.add_argument("output_directory", help="Output directory") + self.parser_icons.add_argument( + "--filename", + help="Base filename for file with icon data", + required=False, + default="assets_icons", + ) + self.parser_icons.set_defaults(func=self.icons) self.parser_manifest = self.subparsers.add_parser( @@ -102,13 +109,15 @@ class Main(App): return extension in ICONS_SUPPORTED_FORMATS def icons(self): - self.logger.debug(f"Converting icons") + self.logger.debug("Converting icons") icons_c = open( - os.path.join(self.args.output_directory, "assets_icons.c"), + os.path.join(self.args.output_directory, f"{self.args.filename}.c"), "w", newline="\n", ) - icons_c.write(ICONS_TEMPLATE_C_HEADER) + icons_c.write( + ICONS_TEMPLATE_C_HEADER.format(assets_filename=self.args.filename) + ) icons = [] # Traverse icons tree, append image data to source file for dirpath, dirnames, filenames in os.walk(self.args.input_directory): @@ -194,7 +203,7 @@ class Main(App): # Create Public Header self.logger.debug(f"Creating header") icons_h = open( - os.path.join(self.args.output_directory, "assets_icons.h"), + os.path.join(self.args.output_directory, f"{self.args.filename}.h"), "w", newline="\n", ) diff --git a/scripts/lint.py b/scripts/lint.py index 30a5699a7..c178c8763 100755 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -54,6 +54,10 @@ class Main(App): output = [] for folder in folders: for dirpath, dirnames, filenames in os.walk(folder): + # Skipping 3rd-party code - usually resides in subfolder "lib" + if "lib" in dirnames: + dirnames.remove("lib") + for filename in filenames: ext = os.path.splitext(filename.lower())[1] if not ext in SOURCE_CODE_FILE_EXTENSIONS: diff --git a/site_scons/commandline.scons b/site_scons/commandline.scons index 2eeda2479..765af08f1 100644 --- a/site_scons/commandline.scons +++ b/site_scons/commandline.scons @@ -34,6 +34,14 @@ AddOption( help="List of applications to forcefully build as standalone .elf", ) +AddOption( + "--proxy-env", + action="store", + dest="proxy_env", + default="", + help="Comma-separated list of additional environment variables to pass to child SCons processes", +) + # Construction environment variables @@ -230,6 +238,7 @@ vars.Add( ("applications/system", False), ("applications/debug", False), ("applications/plugins", False), + ("applications/examples", False), ("applications_user", False), ], ) diff --git a/site_scons/environ.scons b/site_scons/environ.scons index c61f29616..3e0c6bea7 100644 --- a/site_scons/environ.scons +++ b/site_scons/environ.scons @@ -12,14 +12,20 @@ forward_os_env = { "PATH": os.environ["PATH"], } # Proxying CI environment to child processes & scripts -for env_value_name in ( +variables_to_forward = [ "WORKFLOW_BRANCH_OR_TAG", "DIST_SUFFIX", "HOME", "APPDATA", "PYTHONHOME", "PYTHONNOUSERSITE", -): + "TMP", + "TEMP", +] +if proxy_env := GetOption("proxy_env"): + variables_to_forward.extend(proxy_env.split(",")) + +for env_value_name in variables_to_forward: if environ_value := os.environ.get(env_value_name, None): forward_os_env[env_value_name] = environ_value diff --git a/site_scons/extapps.scons b/site_scons/extapps.scons index 66002915f..c976fbbe5 100644 --- a/site_scons/extapps.scons +++ b/site_scons/extapps.scons @@ -1,10 +1,23 @@ +from SCons.Errors import UserError + + Import("ENV") from fbt.appmanifest import FlipperAppType appenv = ENV.Clone( - tools=[("fbt_extapps", {"EXT_APPS_WORK_DIR": ENV.subst("${BUILD_DIR}/.extapps")})] + tools=[ + ( + "fbt_extapps", + { + "EXT_APPS_WORK_DIR": ENV.subst( + "${BUILD_DIR}/.extapps", + ) + }, + ), + "fbt_assets", + ] ) appenv.Replace( @@ -83,7 +96,16 @@ if extra_app_list := GetOption("extra_ext_apps"): if appenv["FORCE"]: appenv.AlwaysBuild(extapps["compact"].values()) -Alias(appenv["FIRMWARE_BUILD_CFG"] + "_extapps", extapps["compact"].values()) + +# Deprecation stub +def legacy_app_build_stub(**kw): + raise UserError(f"Target name 'firmware_extapps' is deprecated, use 'faps' instead") + + +appenv.PhonyTarget("firmware_extapps", appenv.Action(legacy_app_build_stub, None)) + + +Alias("faps", extapps["compact"].values()) if appsrc := appenv.subst("$APPSRC"): app_manifest, fap_file, app_validator = appenv.GetExtAppFromPath(appsrc) diff --git a/site_scons/fbt/appmanifest.py b/site_scons/fbt/appmanifest.py index a1132eeab..de7c6b682 100644 --- a/site_scons/fbt/appmanifest.py +++ b/site_scons/fbt/appmanifest.py @@ -23,6 +23,22 @@ class FlipperAppType(Enum): @dataclass class FlipperApplication: + @dataclass + class ExternallyBuiltFile: + path: str + command: str + + @dataclass + class Library: + name: str + fap_include_paths: List[str] = field(default_factory=lambda: ["."]) + sources: List[str] = field(default_factory=lambda: ["*.c*"]) + cflags: List[str] = field(default_factory=list) + cdefines: List[str] = field(default_factory=list) + cincludes: List[str] = field(default_factory=list) + + PRIVATE_FIELD_PREFIX = "_" + appid: str apptype: FlipperAppType name: Optional[str] = "" @@ -45,6 +61,9 @@ class FlipperApplication: fap_description: str = "" fap_author: str = "" fap_weburl: str = "" + fap_icon_assets: Optional[str] = None + fap_extbuild: List[ExternallyBuiltFile] = field(default_factory=list) + fap_private_libs: List[Library] = field(default_factory=list) # Internally used by fbt _appdir: Optional[object] = None _apppath: Optional[str] = None @@ -88,6 +107,12 @@ class AppManager: ), ) + def ExtFile(*args, **kw): + return FlipperApplication.ExternallyBuiltFile(*args, **kw) + + def Lib(*args, **kw): + return FlipperApplication.Library(*args, **kw) + try: with open(app_manifest_path, "rt") as manifest_file: exec(manifest_file.read()) diff --git a/site_scons/site_tools/fbt_assets.py b/site_scons/site_tools/fbt_assets.py index 877948471..f058d15f9 100644 --- a/site_scons/site_tools/fbt_assets.py +++ b/site_scons/site_tools/fbt_assets.py @@ -10,8 +10,8 @@ import subprocess def icons_emitter(target, source, env): target = [ - "compiled/assets_icons.c", - "compiled/assets_icons.h", + target[0].File(env.subst("${ICON_FILE_NAME}.c")), + target[0].File(env.subst("${ICON_FILE_NAME}.h")), ] source = env.GlobRecursive("*.*", env["ICON_SRC_DIR"]) return target, source @@ -99,17 +99,41 @@ def proto_ver_generator(target, source, env): file.write("\n".join(version_file_data)) +def CompileIcons(env, target_dir, source_dir, *, icon_bundle_name="assets_icons"): + # Gathering icons sources + icons_src = env.GlobRecursive("*.png", source_dir) + icons_src += env.GlobRecursive("frame_rate", source_dir) + + icons = env.IconBuilder( + target_dir, + ICON_SRC_DIR=source_dir, + ICON_FILE_NAME=icon_bundle_name, + ) + env.Depends(icons, icons_src) + return icons + + def generate(env): env.SetDefault( ASSETS_COMPILER="${ROOT_DIR.abspath}/scripts/assets.py", NANOPB_COMPILER="${ROOT_DIR.abspath}/lib/nanopb/generator/nanopb_generator.py", ) + env.AddMethod(CompileIcons) + + if not env["VERBOSE"]: + env.SetDefault( + ICONSCOMSTR="\tICONS\t${TARGET}", + PROTOCOMSTR="\tPROTO\t${SOURCE}", + DOLPHINCOMSTR="\tDOLPHIN\t${DOLPHIN_RES_TYPE}", + RESMANIFESTCOMSTR="\tMANIFEST\t${TARGET}", + PBVERCOMSTR="\tPBVER\t${TARGET}", + ) env.Append( BUILDERS={ "IconBuilder": Builder( action=Action( - '${PYTHON3} "${ASSETS_COMPILER}" icons ${ICON_SRC_DIR} ${TARGET.dir}', + '${PYTHON3} "${ASSETS_COMPILER}" icons ${ICON_SRC_DIR} ${TARGET.dir} --filename ${ICON_FILE_NAME}', "${ICONSCOMSTR}", ), emitter=icons_emitter, diff --git a/site_scons/site_tools/fbt_extapps.py b/site_scons/site_tools/fbt_extapps.py index fec240710..a1fa77140 100644 --- a/site_scons/site_tools/fbt_extapps.py +++ b/site_scons/site_tools/fbt_extapps.py @@ -6,56 +6,146 @@ import SCons.Warnings import os import pathlib from fbt.elfmanifest import assemble_manifest_data +from fbt.appmanifest import FlipperManifestException from fbt.sdk import SdkCache import itertools +from site_scons.fbt.appmanifest import FlipperApplication + def BuildAppElf(env, app): - work_dir = env.subst("$EXT_APPS_WORK_DIR") + ext_apps_work_dir = env.subst("$EXT_APPS_WORK_DIR") + app_work_dir = os.path.join(ext_apps_work_dir, app.appid) + + env.VariantDir(app_work_dir, app._appdir, duplicate=False) + + app_env = env.Clone(FAP_SRC_DIR=app._appdir, FAP_WORK_DIR=app_work_dir) + + app_alias = f"fap_{app.appid}" + + # Deprecation stub + legacy_app_taget_name = f"{app_env['FIRMWARE_BUILD_CFG']}_{app.appid}" + + def legacy_app_build_stub(**kw): + raise UserError( + f"Target name '{legacy_app_taget_name}' is deprecated, use '{app_alias}' instead" + ) + + app_env.PhonyTarget(legacy_app_taget_name, Action(legacy_app_build_stub, None)) + + externally_built_files = [] + if app.fap_extbuild: + for external_file_def in app.fap_extbuild: + externally_built_files.append(external_file_def.path) + app_env.Alias(app_alias, external_file_def.path) + app_env.AlwaysBuild( + app_env.Command( + external_file_def.path, + None, + Action( + external_file_def.command, + "" if app_env["VERBOSE"] else "\tEXTCMD\t${TARGET}", + ), + ) + ) + + if app.fap_icon_assets: + app_env.CompileIcons( + app_env.Dir(app_work_dir), + app._appdir.Dir(app.fap_icon_assets), + icon_bundle_name=f"{app.appid}_icons", + ) + + private_libs = [] + + for lib_def in app.fap_private_libs: + lib_src_root_path = os.path.join(app_work_dir, "lib", lib_def.name) + app_env.AppendUnique( + CPPPATH=list( + app_env.Dir(lib_src_root_path).Dir(incpath).srcnode() + for incpath in lib_def.fap_include_paths + ), + ) + + lib_sources = list( + itertools.chain.from_iterable( + app_env.GlobRecursive(source_type, lib_src_root_path) + for source_type in lib_def.sources + ) + ) + if len(lib_sources) == 0: + raise UserError(f"No sources gathered for private library {lib_def}") + + private_lib_env = app_env.Clone() + private_lib_env.AppendUnique( + CCFLAGS=[ + *lib_def.cflags, + ], + CPPDEFINES=lib_def.cdefines, + CPPPATH=list( + os.path.join(app._appdir.path, cinclude) + for cinclude in lib_def.cincludes + ), + ) + + lib = private_lib_env.StaticLibrary( + os.path.join(app_work_dir, lib_def.name), + lib_sources, + ) + private_libs.append(lib) - app_alias = f"{env['FIRMWARE_BUILD_CFG']}_{app.appid}" - app_original_elf = os.path.join(work_dir, f"{app.appid}_d") app_sources = list( itertools.chain.from_iterable( - env.GlobRecursive(source_type, os.path.join(work_dir, app._appdir.relpath)) + app_env.GlobRecursive( + source_type, + app_work_dir, + exclude="lib", + ) for source_type in app.sources ) ) - app_elf_raw = env.Program( - app_original_elf, - app_sources, - APP_ENTRY=app.entry_point, - LIBS=env["LIBS"] + app.fap_libs, + + app_env.Append( + LIBS=[*app.fap_libs, *private_libs], + CPPPATH=env.Dir(app_work_dir), ) - app_elf_dump = env.ObjDump(app_elf_raw) - env.Alias(f"{app_alias}_list", app_elf_dump) + app_elf_raw = app_env.Program( + os.path.join(app_work_dir, f"{app.appid}_d"), + app_sources, + APP_ENTRY=app.entry_point, + ) - app_elf_augmented = env.EmbedAppMetadata( - os.path.join(env.subst("$PLUGIN_ELF_DIR"), app.appid), + app_env.Clean(app_elf_raw, [*externally_built_files, app_env.Dir(app_work_dir)]) + + app_elf_dump = app_env.ObjDump(app_elf_raw) + app_env.Alias(f"{app_alias}_list", app_elf_dump) + + app_elf_augmented = app_env.EmbedAppMetadata( + os.path.join(ext_apps_work_dir, app.appid), app_elf_raw, APP=app, ) - manifest_vals = vars(app) manifest_vals = { - k: v for k, v in manifest_vals.items() if k not in ("_appdir", "_apppath") + k: v + for k, v in vars(app).items() + if not k.startswith(FlipperApplication.PRIVATE_FIELD_PREFIX) } - env.Depends( + app_env.Depends( app_elf_augmented, - [env["SDK_DEFINITION"], env.Value(manifest_vals)], + [app_env["SDK_DEFINITION"], app_env.Value(manifest_vals)], ) if app.fap_icon: - env.Depends( + app_env.Depends( app_elf_augmented, - env.File(f"{app._apppath}/{app.fap_icon}"), + app_env.File(f"{app._apppath}/{app.fap_icon}"), ) - env.Alias(app_alias, app_elf_augmented) - app_elf_import_validator = env.ValidateAppImports(app_elf_augmented) - env.AlwaysBuild(app_elf_import_validator) - env.Alias(app_alias, app_elf_import_validator) + app_elf_import_validator = app_env.ValidateAppImports(app_elf_augmented) + app_env.AlwaysBuild(app_elf_import_validator) + app_env.Alias(app_alias, app_elf_import_validator) return (app_elf_augmented, app_elf_raw, app_elf_import_validator) @@ -101,9 +191,15 @@ def GetExtAppFromPath(env, app_dir): appmgr = env["APPMGR"] app = None - for dir_part in reversed(pathlib.Path(app_dir).parts): - if app := appmgr.find_by_appdir(dir_part): - break + try: + # Maybe used passed an appid? + app = appmgr.get(app_dir) + except FlipperManifestException as _: + # Look up path components in known app dits + for dir_part in reversed(pathlib.Path(app_dir).parts): + if app := appmgr.find_by_appdir(dir_part): + break + if not app: raise UserError(f"Failed to resolve application for given APPSRC={app_dir}") @@ -120,7 +216,7 @@ def GetExtAppFromPath(env, app_dir): def generate(env, **kw): env.SetDefault(EXT_APPS_WORK_DIR=kw.get("EXT_APPS_WORK_DIR")) - env.VariantDir(env.subst("$EXT_APPS_WORK_DIR"), env.Dir("#"), duplicate=False) + # env.VariantDir(env.subst("$EXT_APPS_WORK_DIR"), env.Dir("#"), duplicate=False) env.AddMethod(BuildAppElf) env.AddMethod(GetExtAppFromPath) From 61189c3c82cbd397bb9a18619c4ad3bce4d9293d Mon Sep 17 00:00:00 2001 From: Georgii Surkov <37121527+gsurkov@users.noreply.github.com> Date: Thu, 6 Oct 2022 17:18:20 +0300 Subject: [PATCH 2/2] [FL-2847] FFF trailing space fix (#1811) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Improve whitespace handlilng in FFF * Add tests for odd fff user input * Adjust formatting Co-authored-by: あく --- .../flipper_format/flipper_format_test.c | 26 ++++++++ lib/flipper_format/flipper_format_stream.c | 63 +++++++++++-------- 2 files changed, 64 insertions(+), 25 deletions(-) diff --git a/applications/debug/unit_tests/flipper_format/flipper_format_test.c b/applications/debug/unit_tests/flipper_format/flipper_format_test.c index ccd5e751f..012e905b6 100644 --- a/applications/debug/unit_tests/flipper_format/flipper_format_test.c +++ b/applications/debug/unit_tests/flipper_format/flipper_format_test.c @@ -57,6 +57,23 @@ static const char* test_data_win = "Filetype: Flipper File test\r\n" "Hex data: DE AD BE"; #define READ_TEST_FLP "ff_flp.test" +#define READ_TEST_ODD "ff_oddities.test" +static const char* test_data_odd = "Filetype: Flipper File test\n" + // Tabs before newline + "Version: 666\t\t\n" + "# This is comment\n" + // Windows newline in a UNIX file + "String data: String\r\n" + // Trailing whitespace + "Int32 data: 1234 -6345 7813 0 \n" + // Extra whitespace + "Uint32 data: 1234 0 5678 9098 7654321 \n" + // Mixed whitespace + "Float data: 1.5\t \t1000.0\n" + // Leading tabs after key + "Bool data:\t\ttrue false\n" + // Mixed trailing whitespace + "Hex data: DE AD BE\t "; // data created by user on linux machine static const char* test_file_linux = TEST_DIR READ_TEST_NIX; @@ -64,6 +81,8 @@ static const char* test_file_linux = TEST_DIR READ_TEST_NIX; static const char* test_file_windows = TEST_DIR READ_TEST_WIN; // data created by flipper itself static const char* test_file_flipper = TEST_DIR READ_TEST_FLP; +// data containing odd user input +static const char* test_file_oddities = TEST_DIR READ_TEST_ODD; static bool storage_write_string(const char* path, const char* data) { Storage* storage = furi_record_open(RECORD_STORAGE); @@ -503,6 +522,12 @@ MU_TEST(flipper_format_multikey_test) { mu_assert(test_read_multikey(TEST_DIR "ff_multiline.test"), "Multikey read test error"); } +MU_TEST(flipper_format_oddities_test) { + mu_assert( + storage_write_string(test_file_oddities, test_data_odd), "Write test error [Oddities]"); + mu_assert(test_read(test_file_linux), "Read test error [Oddities]"); +} + MU_TEST_SUITE(flipper_format) { tests_setup(); MU_RUN_TEST(flipper_format_write_test); @@ -516,6 +541,7 @@ MU_TEST_SUITE(flipper_format) { MU_RUN_TEST(flipper_format_update_2_test); MU_RUN_TEST(flipper_format_update_2_result_test); MU_RUN_TEST(flipper_format_multikey_test); + MU_RUN_TEST(flipper_format_oddities_test); tests_teardown(); } diff --git a/lib/flipper_format/flipper_format_stream.c b/lib/flipper_format/flipper_format_stream.c index ecc68d4ed..41934a3b1 100644 --- a/lib/flipper_format/flipper_format_stream.c +++ b/lib/flipper_format/flipper_format_stream.c @@ -4,6 +4,10 @@ #include "flipper_format_stream.h" #include "flipper_format_stream_i.h" +static inline bool flipper_format_stream_is_space(char c) { + return c == ' ' || c == '\t' || c == flipper_format_eolr; +} + static bool flipper_format_stream_write(Stream* stream, const void* data, size_t data_size) { size_t bytes_written = stream_write(stream, data, data_size); return bytes_written == data_size; @@ -118,55 +122,64 @@ bool flipper_format_stream_seek_to_key(Stream* stream, const char* key, bool str } static bool flipper_format_stream_read_value(Stream* stream, FuriString* value, bool* last) { - furi_string_reset(value); + enum { LeadingSpace, ReadValue, TrailingSpace } state = LeadingSpace; const size_t buffer_size = 32; uint8_t buffer[buffer_size]; bool result = false; bool error = false; + furi_string_reset(value); + while(true) { size_t was_read = stream_read(stream, buffer, buffer_size); if(was_read == 0) { - // check EOF - if(stream_eof(stream) && furi_string_size(value) > 0) { + if(state != LeadingSpace && stream_eof(stream)) { result = true; *last = true; - break; + } else { + error = true; } } for(uint16_t i = 0; i < was_read; i++) { - uint8_t data = buffer[i]; - if(data == flipper_format_eoln) { - if(furi_string_size(value) > 0) { - if(!stream_seek(stream, i - was_read, StreamOffsetFromCurrent)) { - error = true; - break; - } + const uint8_t data = buffer[i]; - result = true; - *last = true; + if(state == LeadingSpace) { + if(flipper_format_stream_is_space(data)) { + continue; + } else if(data == flipper_format_eoln) { + stream_seek(stream, i - was_read, StreamOffsetFromCurrent); + error = true; break; } else { - error = true; + state = ReadValue; + furi_string_push_back(value, data); } - } else if(data == ' ') { - if(furi_string_size(value) > 0) { + } else if(state == ReadValue) { + if(flipper_format_stream_is_space(data)) { + state = TrailingSpace; + } else if(data == flipper_format_eoln) { if(!stream_seek(stream, i - was_read, StreamOffsetFromCurrent)) { error = true; - break; + } else { + result = true; + *last = true; } - - result = true; - *last = false; break; + } else { + furi_string_push_back(value, data); } - - } else if(data == flipper_format_eolr) { - // Ignore - } else { - furi_string_push_back(value, data); + } else if(state == TrailingSpace) { + if(flipper_format_stream_is_space(data)) { + continue; + } else if(!stream_seek(stream, i - was_read, StreamOffsetFromCurrent)) { + error = true; + } else { + *last = (data == flipper_format_eoln); + result = true; + } + break; } }