[core] Native idf full support (#14678)

Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com>
Co-authored-by: Jonathan Swoboda <swoboda1337@users.noreply.github.com>
Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>
Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>

esphome/components/esp32/__init__.py

@@ -31,6 +31,7 @@ from esphome.const import (
     CONF_SAFE_MODE,
     CONF_SIZE,
     CONF_SOURCE,
+    CONF_TOOLCHAIN,
     CONF_TYPE,
     CONF_VARIANT,
     CONF_VERSION,
@@ -38,16 +39,17 @@ from esphome.const import (
     KEY_CORE,
     KEY_FRAMEWORK_VERSION,
     KEY_NAME,
-    KEY_NATIVE_IDF,
     KEY_TARGET_FRAMEWORK,
     KEY_TARGET_PLATFORM,
     PLATFORM_ESP32,
     ThreadModel,
+    Toolchain,
     __version__,
 )
-from esphome.core import CORE, HexInt
+from esphome.core import CORE, HexInt, Library
 from esphome.core.config import BOARD_MAX_LENGTH
 from esphome.coroutine import CoroPriority, coroutine_with_priority
+from esphome.espidf.component import generate_idf_component
 import esphome.final_validate as fv
 from esphome.helpers import copy_file_if_changed, rmtree, write_file_if_changed
 from esphome.types import ConfigType
@@ -465,6 +467,9 @@ def set_core_data(config):
     if conf[CONF_TYPE] == FRAMEWORK_ESP_IDF:
         CORE.data[KEY_ESP32][KEY_IDF_VERSION] = framework_ver
     elif (idf_ver := ARDUINO_IDF_VERSION_LOOKUP.get(framework_ver)) is not None:
+        if CORE.using_toolchain_esp_idf:
+            # Official ESP-IDF frameworks don't use extra
+            idf_ver = cv.Version(idf_ver.major, idf_ver.minor, idf_ver.patch)
         CORE.data[KEY_ESP32][KEY_IDF_VERSION] = idf_ver
     else:
         raise cv.Invalid(
@@ -652,7 +657,7 @@ def _format_framework_arduino_version(ver: cv.Version) -> str:
     return f"{ARDUINO_FRAMEWORK_PKG}@https://github.com/espressif/arduino-esp32/releases/download/{ver}/{filename}"
 
 
-def _format_framework_espidf_version(
+def _format_framework_pio_espidf_version(
     ver: cv.Version, release: str | None = None
 ) -> str:
     # format the given espidf (https://github.com/pioarduino/esp-idf/releases) version to
@@ -741,6 +746,7 @@ ESP_IDF_FRAMEWORK_VERSION_LOOKUP = {
     "latest": cv.Version(5, 5, 4),
     "dev": cv.Version(5, 5, 4),
 }
+
 ESP_IDF_PLATFORM_VERSION_LOOKUP = {
     cv.Version(
         6, 0, 1
@@ -774,7 +780,7 @@ PLATFORM_VERSION_LOOKUP = {
 }
 
 
-def _check_versions(config):
+def _check_pio_versions(config):
     config = config.copy()
     value = config[CONF_FRAMEWORK]
 
@@ -785,7 +791,7 @@ def _check_versions(config):
             )
 
         platform_lookup = PLATFORM_VERSION_LOOKUP[value[CONF_VERSION]]
-        value[CONF_PLATFORM_VERSION] = _parse_platform_version(str(platform_lookup))
+        value[CONF_PLATFORM_VERSION] = _parse_pio_platform_version(str(platform_lookup))
 
         if value[CONF_TYPE] == FRAMEWORK_ARDUINO:
             version = ARDUINO_FRAMEWORK_VERSION_LOOKUP[value[CONF_VERSION]]
@@ -813,7 +819,7 @@ def _check_versions(config):
         platform_lookup = ESP_IDF_PLATFORM_VERSION_LOOKUP.get(version)
         value[CONF_SOURCE] = value.get(
             CONF_SOURCE,
-            _format_framework_espidf_version(version, value.get(CONF_RELEASE)),
+            _format_framework_pio_espidf_version(version, value.get(CONF_RELEASE)),
         )
         if _is_framework_url(value[CONF_SOURCE]):
             value[CONF_SOURCE] = f"pioarduino/framework-espidf@{value[CONF_SOURCE]}"
@@ -823,7 +829,7 @@ def _check_versions(config):
             raise cv.Invalid(
                 "Framework version not recognized; please specify platform_version"
             )
-        value[CONF_PLATFORM_VERSION] = _parse_platform_version(str(platform_lookup))
+        value[CONF_PLATFORM_VERSION] = _parse_pio_platform_version(str(platform_lookup))
 
     if version != recommended_version:
         _LOGGER.warning(
@@ -831,7 +837,7 @@ def _check_versions(config):
             "If there are connectivity or build issues please remove the manual version."
         )
 
-    if value[CONF_PLATFORM_VERSION] != _parse_platform_version(
+    if value[CONF_PLATFORM_VERSION] != _parse_pio_platform_version(
         str(PLATFORM_VERSION_LOOKUP["recommended"])
     ):
         _LOGGER.warning(
@@ -842,7 +848,38 @@ def _check_versions(config):
     return config
 
 
-def _parse_platform_version(value):
+def _check_esp_idf_versions(config):
+    config = _check_pio_versions(config)
+    value = config[CONF_FRAMEWORK]
+
+    # Remove unwanted keys if present
+    for key in (CONF_SOURCE, CONF_PLATFORM_VERSION):
+        value.pop(key, None)
+
+    # Official ESP-IDF frameworks don't use extra
+    version = cv.Version.parse(value[CONF_VERSION])
+    version = cv.Version(version.major, version.minor, version.patch)
+
+    value[CONF_VERSION] = str(version)
+
+    return config
+
+
+def _validate_toolchain(value) -> Toolchain:
+    return Toolchain(cv.one_of(*(t.value for t in Toolchain), lower=True)(value))
+
+
+def _check_versions(config):
+    # Resolve toolchain: CLI (already on CORE.toolchain) > YAML > default.
+    if CORE.toolchain is None:
+        CORE.toolchain = config.get(CONF_TOOLCHAIN, Toolchain.PLATFORMIO)
+
+    if CORE.using_toolchain_esp_idf:
+        return _check_esp_idf_versions(config)
+    return _check_pio_versions(config)
+
+
+def _parse_pio_platform_version(value):
     try:
         ver = cv.Version.parse(cv.version_number(value))
         release = f"{ver.major}.{ver.minor:02d}.{ver.patch:02d}"
@@ -1272,7 +1309,7 @@ FRAMEWORK_SCHEMA = cv.Schema(
         cv.Optional(CONF_VERSION, default="recommended"): cv.string_strict,
         cv.Optional(CONF_RELEASE): cv.string_strict,
         cv.Optional(CONF_SOURCE): cv.string_strict,
-        cv.Optional(CONF_PLATFORM_VERSION): _parse_platform_version,
+        cv.Optional(CONF_PLATFORM_VERSION): _parse_pio_platform_version,
         cv.Optional(CONF_SDKCONFIG_OPTIONS, default={}): {
             cv.string_strict: cv.string_strict
         },
@@ -1524,6 +1561,7 @@ CONFIG_SCHEMA = cv.All(
             ),
             cv.Optional(CONF_VARIANT): cv.one_of(*VARIANTS, upper=True),
             cv.Optional(CONF_FRAMEWORK): FRAMEWORK_SCHEMA,
+            cv.Optional(CONF_TOOLCHAIN): _validate_toolchain,
             cv.Optional(CONF_WATCHDOG_TIMEOUT, default="5s"): cv.All(
                 cv.positive_time_period_seconds,
                 cv.Range(min=cv.TimePeriod(seconds=5), max=cv.TimePeriod(seconds=60)),
@@ -1672,11 +1710,11 @@ async def to_code(config):
     framework_ver: cv.Version = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]
     conf = config[CONF_FRAMEWORK]
 
-    # Check if using native ESP-IDF build (--native-idf)
-    use_platformio = not CORE.data.get(KEY_NATIVE_IDF, False)
+    # Check if using ESP-IDF toolchain
+    use_platformio = not CORE.using_toolchain_esp_idf
     if use_platformio:
         # Clear IDF environment variables to avoid conflicts with PlatformIO's ESP-IDF
-        # but keep them when using --native-idf for native ESP-IDF builds
+        # but keep them when using ESP-IDF toolchain
         for clean_var in ("IDF_PATH", "IDF_TOOLS_PATH"):
             os.environ.pop(clean_var, None)
 
@@ -1716,6 +1754,8 @@ async def to_code(config):
             )
     else:
         cg.add_build_flag("-Wno-error=format")
+        cg.add_build_flag("-Wno-error=missing-field-initializers")
+        cg.add_build_flag("-Wno-error=volatile")
 
     cg.set_cpp_standard("gnu++20")
     cg.add_build_flag("-DUSE_ESP32")
@@ -1792,7 +1832,7 @@ async def to_code(config):
             if (idf_ver := ARDUINO_IDF_VERSION_LOOKUP.get(framework_ver)) is not None:
                 cg.add_platformio_option(
                     "platform_packages",
-                    [_format_framework_espidf_version(idf_ver)],
+                    [_format_framework_pio_espidf_version(idf_ver)],
                 )
                 # Use stub package to skip downloading precompiled libs
                 stubs_dir = CORE.relative_build_path("arduino_libs_stub")
@@ -2424,6 +2464,14 @@ def _write_sdkconfig():
         clean_build(clear_pio_cache=False)
 
 
+def _platformio_library_to_dependency(library: Library) -> tuple[str, dict[str, str]]:
+    dependency: dict[str, str] = {}
+    name, version, path = generate_idf_component(library)
+    dependency["override_path"] = str(path)
+    dependency["version"] = version
+    return name, dependency
+
+
 def _write_idf_component_yml():
     yml_path = CORE.relative_build_path("src/idf_component.yml")
     dependencies: dict[str, dict] = {}
@@ -2465,6 +2513,21 @@ def _write_idf_component_yml():
             if stub_path.exists():
                 rmtree(stub_path)
 
+        if CORE.using_toolchain_esp_idf:
+            add_idf_component(
+                name="espressif/arduino-esp32",
+                ref=str(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]),
+            )
+
+    if CORE.using_toolchain_esp_idf:
+        # Try to convert PlatformIO library to ESP-IDF components
+        for name, library in CORE.platformio_libraries.items():
+            # Don't process arduino libraries
+            if name in ARDUINO_DISABLED_LIBRARIES:
+                continue
+            dependency_name, dependency = _platformio_library_to_dependency(library)
+            dependencies[dependency_name] = dependency
+
     if CORE.data[KEY_ESP32][KEY_COMPONENTS]:
         components: dict = CORE.data[KEY_ESP32][KEY_COMPONENTS]
         for name, component in components.items():