diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 6618c9cd2..71cce2923 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -24,7 +24,9 @@ import json import subprocess import sys +import shutil import os +import pkg_resources import click import semantic_version @@ -36,11 +38,17 @@ ) from platformio import fs +from platformio.compat import IS_WINDOWS from platformio.proc import exec_command -from platformio.util import get_systype from platformio.builder.tools.piolib import ProjectAsLibBuilder from platformio.package.version import get_original_version, pepver_to_semver +# Added to avoid conflicts between installed Python packages from +# the IDF virtual environment and PlatformIO Core +# Note: This workaround can be safely deleted when PlatformIO 6.1.7 is released +if os.environ.get("PYTHONPATH"): + del os.environ["PYTHONPATH"] + env = DefaultEnvironment() env.SConscript("_embed_files.py", exports="env") @@ -49,6 +57,9 @@ mcu = board.get("build.mcu", "esp32") idf_variant = mcu.lower() +# Required until Arduino switches to v5 +IDF5 = platform.get_package_version("framework-espidf").split(".")[1].startswith("5") +IDF_ENV_VERSION = "1.0.0" FRAMEWORK_DIR = platform.get_package_dir("framework-espidf") TOOLCHAIN_DIR = platform.get_package_dir( "toolchain-%s" % ("riscv32-esp" if mcu == "esp32c3" else ("xtensa-%s" % mcu)) @@ -75,10 +86,10 @@ PROJECT_DIR = env.subst("$PROJECT_DIR") PROJECT_SRC_DIR = env.subst("$PROJECT_SRC_DIR") CMAKE_API_REPLY_PATH = os.path.join(".cmake", "api", "v1", "reply") -SDKCONFIG_PATH = board.get( +SDKCONFIG_PATH = os.path.expandvars(board.get( "build.esp-idf.sdkconfig_path", os.path.join(PROJECT_DIR, "sdkconfig.%s" % env.subst("$PIOENV")), -) +)) def get_project_lib_includes(env): @@ -220,7 +231,7 @@ def populate_idf_env_vars(idf_env): os.path.join(TOOLCHAIN_DIR, "bin"), platform.get_package_dir("tool-ninja"), os.path.join(platform.get_package_dir("tool-cmake"), "bin"), - os.path.dirname(env.subst("$PYTHONEXE")), + os.path.dirname(get_python_exe()), ] if mcu != "esp32c3": @@ -228,7 +239,7 @@ def populate_idf_env_vars(idf_env): os.path.join(platform.get_package_dir("toolchain-esp32ulp"), "bin"), ) - if "windows" in get_systype(): + if IS_WINDOWS: additional_packages.append(platform.get_package_dir("tool-mconf")) idf_env["PATH"] = os.pathsep.join(additional_packages + [idf_env["PATH"]]) @@ -293,16 +304,25 @@ def get_app_includes(app_config): def extract_defines(compile_group): - result = [] - result.extend( - [ - d.get("define").replace('"', '\\"').strip() - for d in compile_group.get("defines", []) - ] - ) + def _normalize_define(define_string): + define_string = define_string.strip() + if "=" in define_string: + define, value = define_string.split("=", maxsplit=1) + if '"' in value and not value.startswith("\\"): + # Escape only raw values + value = value.replace('"', '\\"') + return (define, value) + return define_string + + result = [ + _normalize_define(d.get("define", "")) + for d in compile_group.get("defines", []) if d + ] + for f in compile_group.get("compileCommandFragments", []): if f.get("fragment", "").startswith("-D"): - result.append(f["fragment"][2:]) + result.append(_normalize_define(f["fragment"][2:])) + return result @@ -555,7 +575,7 @@ def generate_project_ld_script(sdk_config, ignore_targets=None): } cmd = ( - '"$PYTHONEXE" "{script}" --input $SOURCE ' + '"$ESPIDF_PYTHONEXE" "{script}" --input $SOURCE ' '--config "{config}" --fragments {fragments} --output $TARGET ' '--kconfig "{kconfig}" --env-file "{env_file}" ' '--libraries-file "{libraries_list}" ' @@ -576,6 +596,16 @@ def generate_project_ld_script(sdk_config, ignore_targets=None): ) +# A temporary workaround to avoid modifying CMake mainly for the "heap" library. +# The "tlsf.c" source file in this library has an include flag relative +# to CMAKE_CURRENT_SOURCE_DIR which breaks PlatformIO builds that have a +# different working directory +def _fix_component_relative_include(config, build_flags, source_index): + source_file_path = config["sources"][source_index]["path"] + build_flags = build_flags.replace("..", os.path.dirname(source_file_path) + "/..") + return build_flags + + def prepare_build_envs(config, default_env, debug_allowed=True): build_envs = [] target_compile_groups = config.get("compileGroups") @@ -597,7 +627,14 @@ def prepare_build_envs(config, default_env, debug_allowed=True): for cc in compile_commands: build_flags = cc.get("fragment") if not build_flags.startswith("-D"): - build_env.AppendUnique(**build_env.ParseFlags(build_flags)) + if build_flags.startswith("-include") and ".." in build_flags: + source_index = cg.get("sourceIndexes")[0] + build_flags = _fix_component_relative_include( + config, build_flags, source_index) + parsed_flags = build_env.ParseFlags(build_flags) + build_env.AppendUnique(**parsed_flags) + if cg.get("language", "") == "ASM": + build_env.AppendUnique(ASFLAGS=parsed_flags.get("CCFLAGS", [])) build_env.AppendUnique(CPPDEFINES=defines, CPPPATH=includes) if sys_includes: build_env.Append(CCFLAGS=[("-isystem", inc) for inc in sys_includes]) @@ -639,9 +676,17 @@ def compile_source_files( else: obj_path = os.path.join(obj_path, os.path.basename(src_path)) + preserve_source_file_extension = board.get( + "build.esp-idf.preserve_source_file_extension", False + ) + objects.append( build_envs[compile_group_idx].StaticObject( - target=os.path.splitext(obj_path)[0] + ".o", + target=( + obj_path + if preserve_source_file_extension + else os.path.splitext(obj_path)[0] + ) + ".o", source=os.path.realpath(src_path), ) ) @@ -737,7 +782,7 @@ def build_bootloader(sdk_config): [ "-DIDF_TARGET=" + idf_variant, "-DPYTHON_DEPS_CHECKED=1", - "-DPYTHON=" + env.subst("$PYTHONEXE"), + "-DPYTHON=" + get_python_exe(), "-DIDF_PATH=" + FRAMEWORK_DIR, "-DSDKCONFIG=" + SDKCONFIG_PATH, "-DLEGACY_INCLUDE_COMMON_HEADERS=", @@ -892,7 +937,7 @@ def generate_empty_partition_image(binary_path, image_size): binary_path, None, env.VerboseAction( - '"$PYTHONEXE" "%s" %s $TARGET' + '"$ESPIDF_PYTHONEXE" "%s" %s $TARGET' % ( os.path.join( FRAMEWORK_DIR, @@ -917,7 +962,7 @@ def get_partition_info(pt_path, pt_offset, pt_params): env.Exit(1) cmd = [ - env.subst("$PYTHONEXE"), + get_python_exe(), os.path.join(FRAMEWORK_DIR, "components", "partition_table", "parttool.py"), "-q", "--partition-table-offset", @@ -974,13 +1019,15 @@ def generate_mbedtls_bundle(sdk_config): FRAMEWORK_DIR, "components", "mbedtls", "esp_crt_bundle" ) - cmd = [env.subst("$PYTHONEXE"), os.path.join(default_crt_dir, "gen_crt_bundle.py")] + cmd = [get_python_exe(), os.path.join(default_crt_dir, "gen_crt_bundle.py")] crt_args = ["--input"] if sdk_config.get("MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL", False): crt_args.append(os.path.join(default_crt_dir, "cacrt_all.pem")) + crt_args.append(os.path.join(default_crt_dir, "cacrt_local.pem")) elif sdk_config.get("MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN", False): crt_args.append(os.path.join(default_crt_dir, "cacrt_all.pem")) + crt_args.append(os.path.join(default_crt_dir, "cacrt_local.pem")) cmd.extend( ["--filter", os.path.join(default_crt_dir, "cmn_crt_authorities.csv")] ) @@ -1025,11 +1072,18 @@ def generate_mbedtls_bundle(sdk_config): def install_python_deps(): - def _get_installed_pip_packages(): + def _get_installed_pip_packages(python_exe_path): result = {} packages = {} pip_output = subprocess.check_output( - [env.subst("$PYTHONEXE"), "-m", "pip", "list", "--format=json"] + [ + python_exe_path, + "-m", + "pip", + "list", + "--format=json", + "--disable-pip-version-check", + ] ) try: packages = json.loads(pip_output) @@ -1042,20 +1096,23 @@ def _get_installed_pip_packages(): return result deps = { + # https://github.com/platformio/platformio-core/issues/4614 + "urllib3": "<2", # https://github.com/platformio/platform-espressif32/issues/635 - "cryptography": ">=2.1.4,<35.0.0", - "future": ">=0.15.2", - "pyparsing": ">=2.0.3,<2.4.0", - "kconfiglib": "==13.7.1", - "idf-component-manager": "~=1.0" + "cryptography": "~=41.0.1" if IDF5 else ">=2.1.4,<35.0.0", + "future": ">=0.18.3", + "pyparsing": "~=3.0.9" if IDF5 else ">=2.0.3,<2.4.0", + "kconfiglib": "~=14.1.0" if IDF5 else "~=13.7.1", + "idf-component-manager": "~=1.2.3" if IDF5 else "~=1.0", } - installed_packages = _get_installed_pip_packages() + python_exe_path = get_python_exe() + installed_packages = _get_installed_pip_packages(python_exe_path) packages_to_install = [] for package, spec in deps.items(): if package not in installed_packages: packages_to_install.append(package) - else: + elif spec: version_spec = semantic_version.Spec(spec) if not version_spec.match(installed_packages[package]): packages_to_install.append(package) @@ -1064,31 +1121,115 @@ def _get_installed_pip_packages(): env.Execute( env.VerboseAction( ( - '"$PYTHONEXE" -m pip install -U --force-reinstall ' + '"%s" -m pip install -U ' % python_exe_path + " ".join(['"%s%s"' % (p, deps[p]) for p in packages_to_install]) ), "Installing ESP-IDF's Python dependencies", ) ) - # a special "esp-windows-curses" python package is required on Windows for Menuconfig - if "windows" in get_systype(): - import pkg_resources + if IS_WINDOWS and "windows-curses" not in installed_packages: + env.Execute( + env.VerboseAction( + '"%s" -m pip install windows-curses' % python_exe_path, + "Installing windows-curses package", + ) + ) - if "esp-windows-curses" not in {pkg.key for pkg in pkg_resources.working_set}: + # A special "esp-windows-curses" python package is required on Windows + # for Menuconfig on IDF <5 + if not IDF5 and "esp-windows-curses" not in { + pkg.key for pkg in pkg_resources.working_set + }: env.Execute( env.VerboseAction( - '$PYTHONEXE -m pip install "file://%s/tools/kconfig_new/esp-windows-curses" windows-curses' - % FRAMEWORK_DIR, + '"%s" -m pip install "file://%s/tools/kconfig_new/esp-windows-curses"' + % (python_exe_path, FRAMEWORK_DIR), "Installing windows-curses package", ) ) +def get_idf_venv_dir(): + # The name of the IDF venv contains the IDF version to avoid possible conflicts and + # unnecessary reinstallation of Python dependencies in cases when Arduino + # as an IDF component requires a different version of the IDF package and + # hence a different set of Python deps or their versions + idf_version = get_original_version(platform.get_package_version("framework-espidf")) + return os.path.join( + env.subst("$PROJECT_CORE_DIR"), "penv", ".espidf-" + idf_version + ) + +def ensure_python_venv_available(): + + def _is_venv_outdated(venv_data_file): + try: + with open(venv_data_file, "r", encoding="utf8") as fp: + venv_data = json.load(fp) + if venv_data.get("version", "") != IDF_ENV_VERSION: + return True + return False + except: + return True + + def _create_venv(venv_dir): + pip_path = os.path.join( + venv_dir, + "Scripts" if IS_WINDOWS else "bin", + "pip" + (".exe" if IS_WINDOWS else ""), + ) + + if os.path.isdir(venv_dir): + try: + print("Removing an oudated IDF virtual environment") + shutil.rmtree(venv_dir) + except OSError: + print( + "Error: Cannot remove an outdated IDF virtual environment. " \ + "Please remove the `%s` folder manually!" % venv_dir + ) + env.Exit(1) + + # Use the built-in PlatformIO Python to create a standalone IDF virtual env + env.Execute( + env.VerboseAction( + '"$PYTHONEXE" -m venv --clear "%s"' % venv_dir, + "Creating a new virtual environment for IDF Python dependencies", + ) + ) + + assert os.path.isfile( + pip_path + ), "Error: Failed to create a proper virtual environment. Missing the `pip` binary!" + + + venv_dir = get_idf_venv_dir() + venv_data_file = os.path.join(venv_dir, "pio-idf-venv.json") + if not os.path.isfile(venv_data_file) or _is_venv_outdated(venv_data_file): + _create_venv(venv_dir) + with open(venv_data_file, "w", encoding="utf8") as fp: + venv_info = {"version": IDF_ENV_VERSION} + json.dump(venv_info, fp, indent=2) + + +def get_python_exe(): + python_exe_path = os.path.join( + get_idf_venv_dir(), + "Scripts" if IS_WINDOWS else "bin", + "python" + (".exe" if IS_WINDOWS else ""), + ) + + assert os.path.isfile(python_exe_path), ( + "Error: Missing Python executable file `%s`" % python_exe_path + ) + + return python_exe_path + # -# ESP-IDF requires Python packages with specific versions +# ESP-IDF requires Python packages with specific versions in a virtual environment # +ensure_python_venv_available() install_python_deps() # ESP-IDF package doesn't contain .git folder, instead package version is specified @@ -1176,6 +1317,10 @@ def _get_installed_pip_packages(): "the `variant` field! The default `esp32` variant will be used." ) extra_components.append(ARDUINO_FRAMEWORK_DIR) + # Add path to internal Arduino libraries so that the LDF will be able to find them + env.Append( + LIBSOURCE_DIRS=[os.path.join(ARDUINO_FRAMEWORK_DIR, "libraries")] + ) print("Reading CMake configuration...") project_codemodel = get_cmake_code_model( @@ -1185,7 +1330,7 @@ def _get_installed_pip_packages(): "-DIDF_TARGET=" + idf_variant, "-DPYTHON_DEPS_CHECKED=1", "-DEXTRA_COMPONENT_DIRS:PATH=" + ";".join(extra_components), - "-DPYTHON=" + env.subst("$PYTHONEXE"), + "-DPYTHON=" + get_python_exe(), "-DSDKCONFIG=" + SDKCONFIG_PATH, ] + click.parser.split_arg_string(board.get("build.cmake_extra_args", "")), @@ -1323,7 +1468,7 @@ def _skip_prj_source_files(node): os.path.join("$BUILD_DIR", "partitions.bin"), "$PARTITIONS_TABLE_CSV", env.VerboseAction( - '"$PYTHONEXE" "%s" -q --offset "%s" --flash-size "%s" $SOURCE $TARGET' + '"$ESPIDF_PYTHONEXE" "%s" -q --offset "%s" --flash-size "%s" $SOURCE $TARGET' % ( os.path.join( FRAMEWORK_DIR, "components", "partition_table", "gen_esp32part.py" @@ -1346,6 +1491,7 @@ def _skip_prj_source_files(node): env.Prepend( CPPPATH=app_includes["plain_includes"], CPPDEFINES=project_defines, + ESPIDF_PYTHONEXE=get_python_exe(), LINKFLAGS=extra_flags, LIBS=libs, FLASH_EXTRA_IMAGES=[ @@ -1471,4 +1617,6 @@ def _skip_prj_source_files(node): ) # Propagate application offset to debug configurations -env["INTEGRATION_EXTRA_DATA"].update({"application_offset": env.subst("$ESP32_APP_OFFSET")}) +env["INTEGRATION_EXTRA_DATA"].update( + {"application_offset": env.subst("$ESP32_APP_OFFSET")} +)