[espidf] Derive idedata from the native ESP-IDF compile_commands.json (#16742)

esphome/__main__.py

@@ -760,6 +760,7 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int:
         toolchain.create_factory_bin()
         toolchain.create_ota_bin()
         toolchain.create_elf_copy()
+        toolchain.get_idedata()
     else:
         from esphome.platformio import toolchain
 

esphome/espidf/idedata.py

@@ -0,0 +1,178 @@
+"""Derive idedata from an ESP-IDF native-toolchain ``compile_commands.json``.
+
+PlatformIO exposes a curated ``pio run -t idedata`` JSON; the native ESP-IDF
+toolchain has no such command, but its CMake build emits
+``build/compile_commands.json`` (CMAKE_EXPORT_COMPILE_COMMANDS). This module
+turns that file into the same fields consumers (IDE integration, clang-tidy)
+expect:
+
+    {cxx_path, cxx_flags, defines, includes: {build, toolchain}}
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from pathlib import Path
+import shlex
+import subprocess
+
+_LOGGER = logging.getLogger(__name__)
+
+# C++ translation-unit suffixes used to identify ESPHome source files.
+_CXX_SUFFIXES = (".cpp", ".cc")
+# Suffixes of input/output files that appear bare on the command line (and so
+# must not be mistaken for compiler flags).
+_INPUT_FILE_SUFFIXES = (*_CXX_SUFFIXES, ".c", ".o", ".S", ".s")
+# Path marker identifying an ESPHome source translation unit.
+_ESPHOME_SRC_MARKER = "/src/esphome/"
+
+
+def _expand_response_files(tokens: list[str], directory: Path) -> list[str]:
+    """Inline any ``@response-file`` arguments (paths relative to ``directory``).
+
+    GCC response files embed flags that must be expanded so GCC-only flags
+    inside them (e.g. ``-mlongcalls``) can be filtered downstream; left as
+    ``@file`` clang would read them and choke.
+    """
+    out: list[str] = []
+    for tok in tokens:
+        if tok.startswith("@"):
+            rf = Path(tok[1:])
+            if not rf.is_absolute():
+                rf = directory / rf
+            try:
+                out.extend(
+                    _expand_response_files(
+                        shlex.split(rf.read_text(encoding="utf-8")), directory
+                    )
+                )
+                continue
+            except OSError as err:
+                # Keep the literal token if the file can't be read, but log it
+                # so the (otherwise opaque) downstream clang failure is traceable.
+                _LOGGER.warning("Could not read response file %s: %s", rf, err)
+        out.append(tok)
+    return out
+
+
+def _pick_entry(entries: list[dict]) -> dict:
+    """Pick a representative ESPHome C++ translation unit.
+
+    All ESPHome sources share the same component flags/defines, so any one of
+    them yields the cxx_path / cxx_flags / defines we need.
+    """
+    for entry in entries:
+        f = entry["file"]
+        if _ESPHOME_SRC_MARKER in f and f.endswith(_CXX_SUFFIXES):
+            return entry
+    for entry in entries:
+        if entry["file"].endswith(_CXX_SUFFIXES):
+            return entry
+    raise ValueError("no C++ translation unit found in compile_commands.json")
+
+
+def _parse_entry(entry: dict) -> tuple[str, list[str], list[str], list[str]]:
+    """Parse one compile_commands entry -> (cxx_path, defines, includes, cxx_flags)."""
+    directory = Path(entry["directory"])
+    tokens = _expand_response_files(shlex.split(entry["command"]), directory)
+
+    def _include(raw: str) -> str:
+        # Include paths in compile_commands are interpreted relative to the
+        # entry's ``directory`` (e.g. build-local ``-Iconfig``); resolve them
+        # so the cached idedata is usable regardless of the consumer's cwd.
+        raw = raw.strip()
+        if raw and not Path(raw).is_absolute():
+            raw = os.path.normpath(directory / raw)
+        return raw
+
+    cxx_path = tokens[0]
+    defines: list[str] = []
+    includes: list[str] = []
+    cxx_flags: list[str] = []
+
+    it = iter(tokens[1:])
+    for tok in it:
+        if tok in ("-c", "-o"):
+            next(it, None)  # drop the flag and its argument (input/output)
+        elif tok.startswith("-D"):
+            # ``.strip()`` handles tokens like ``-D CONFIGURED=1`` (a single
+            # quoted arg with a space after -D) that some flags arrive as.
+            defines.append(tok[2:].strip() if len(tok) > 2 else next(it, "").strip())
+        elif tok.startswith("-I"):
+            includes.append(_include(tok[2:] if len(tok) > 2 else next(it, "")))
+        elif tok == "-isystem":
+            includes.append(_include(next(it, "")))
+        elif tok.startswith("-isystem"):
+            includes.append(_include(tok[len("-isystem") :]))
+        elif tok in ("-MT", "-MF", "-MQ"):
+            next(it, None)  # dependency-file flag + its argument
+        elif tok.startswith(("-MD", "-MMD", "-MP", "-MM")):
+            pass  # dependency-generation flags, no argument
+        elif tok.endswith(_INPUT_FILE_SUFFIXES):
+            pass  # input/output files
+        else:
+            cxx_flags.append(tok)
+    return cxx_path, defines, includes, cxx_flags
+
+
+def _get_toolchain_includes(cxx_path: str) -> list[str]:
+    """Query the compiler for its builtin ``#include <...>`` search dirs."""
+    result = subprocess.run(
+        [cxx_path, "-E", "-x", "c++", "-", "-v"],
+        input="",
+        text=True,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.PIPE,
+        check=False,
+        close_fds=False,
+    )
+    includes: list[str] = []
+    capture = False
+    for line in result.stderr.splitlines():
+        if "#include <...> search starts here:" in line:
+            capture = True
+            continue
+        if "End of search list." in line:
+            break
+        if capture:
+            includes.append(line.strip())
+    if result.returncode != 0 or not includes:
+        raise RuntimeError(
+            f"Could not query builtin include dirs from {cxx_path} "
+            f"(return code {result.returncode}); stderr:\n{result.stderr.strip()}"
+        )
+    return includes
+
+
+def idedata_from_build(compile_commands: Path) -> dict:
+    """Parse compile_commands.json into the idedata fields consumers expect.
+
+    A single ESP-IDF compile entry only carries its own component's REQUIRES
+    include set, but consumers (clang-tidy) analyze ESPHome headers that
+    transitively pull in other components. So take cxx_path / cxx_flags /
+    defines from a representative ESPHome TU, but union the include dirs across
+    all ESPHome TUs to get a project-wide superset (as PlatformIO's idedata
+    provides).
+    """
+    entries = json.loads(Path(compile_commands).read_text(encoding="utf-8"))
+    cxx_path, defines, _, cxx_flags = _parse_entry(_pick_entry(entries))
+
+    build_includes: dict[str, None] = {}
+    for entry in entries:
+        f = entry["file"]
+        if _ESPHOME_SRC_MARKER not in f or not f.endswith(_CXX_SUFFIXES):
+            continue
+        for inc in _parse_entry(entry)[2]:
+            build_includes.setdefault(inc, None)
+
+    return {
+        "cxx_path": cxx_path,
+        "cxx_flags": cxx_flags,
+        "defines": defines,
+        "includes": {
+            "build": list(build_includes),
+            "toolchain": _get_toolchain_includes(cxx_path),
+        },
+    }

esphome/espidf/toolchain.py

@@ -443,6 +443,34 @@ def get_addr2line_path() -> Path:
     return _get_cmake_tool_path("CMAKE_ADDR2LINE")
 
 
+def get_idedata() -> dict | None:
+    """Derive idedata from the build's compile_commands.json.
+
+    The native ESP-IDF toolchain has no ``pio run -t idedata`` equivalent, but
+    its CMake build emits ``build/compile_commands.json``. Parse that into the
+    idedata fields IDE integrations and clang-tidy expect, cached alongside the
+    PlatformIO idedata path. Returns None if the compile DB doesn't exist yet.
+    """
+    from esphome.espidf.idedata import idedata_from_build
+
+    compile_commands = CORE.relative_build_path("build", "compile_commands.json")
+    if not compile_commands.is_file():
+        _LOGGER.debug("No %s yet; skipping idedata generation", compile_commands)
+        return None
+
+    cache = CORE.relative_internal_path("idedata", f"{CORE.name}.json")
+    if cache.is_file() and cache.stat().st_mtime >= compile_commands.stat().st_mtime:
+        try:
+            return json.loads(cache.read_text(encoding="utf-8"))
+        except ValueError:
+            pass
+
+    data = idedata_from_build(compile_commands)
+    cache.parent.mkdir(parents=True, exist_ok=True)
+    cache.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+    return data
+
+
 def create_factory_bin() -> bool:
     """Create factory.bin by merging bootloader, partition table, and app."""
     build_dir = CORE.relative_build_path("build")

tests/unit_tests/test_espidf_idedata.py

@@ -0,0 +1,196 @@
+"""Tests for esphome.espidf.idedata (compile_commands.json -> idedata)."""
+
+# pylint: disable=protected-access
+
+import json
+import os
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from esphome.espidf import idedata
+
+# An absolute, forward-slash (shlex-safe) path prefix valid on the host OS, so
+# tests exercise the same is-absolute / normalize behavior as a real compile DB
+# (a drive-qualified path on Windows, a leading slash elsewhere).
+ABS = "C:/" if os.name == "nt" else "/"
+
+
+def _entry(directory: str, file: str, command: str) -> dict:
+    return {"directory": directory, "file": file, "command": command}
+
+
+def test_parse_entry_extracts_fields() -> None:
+    """cxx_path, defines, includes and remaining flags are split apart."""
+    entry = _entry(
+        f"{ABS}build",
+        f"{ABS}build/src/esphome/core/application.cpp",
+        f"/tools/xtensa-esp32-elf-g++ -DUSE_ESP32 -DESPHOME_LOG_LEVEL=5 "
+        f"-I{ABS}inc/a -isystem {ABS}sys/b -std=gnu++20 -c app.cpp -o app.cpp.o",
+    )
+
+    cxx_path, defines, includes, cxx_flags = idedata._parse_entry(entry)
+
+    assert cxx_path == "/tools/xtensa-esp32-elf-g++"
+    assert "USE_ESP32" in defines
+    assert "ESPHOME_LOG_LEVEL=5" in defines
+    assert f"{ABS}inc/a" in includes
+    assert f"{ABS}sys/b" in includes
+    assert "-std=gnu++20" in cxx_flags
+    # input/output files and their flags are not treated as flags
+    assert "-c" not in cxx_flags
+    assert "-o" not in cxx_flags
+    assert "app.cpp" not in cxx_flags
+    assert "app.cpp.o" not in cxx_flags
+
+
+def test_parse_entry_space_separated_args() -> None:
+    """``-D X`` / ``-I path`` (separate arg) and ``-isystem<path>`` (joined)."""
+    entry = _entry(
+        f"{ABS}build",
+        f"{ABS}build/src/esphome/x.cpp",
+        f"g++ -D FOO=1 -I {ABS}inc/sep -isystem{ABS}sys/joined -c x.cpp",
+    )
+
+    _, defines, includes, _ = idedata._parse_entry(entry)
+
+    assert "FOO=1" in defines
+    assert f"{ABS}inc/sep" in includes
+    assert f"{ABS}sys/joined" in includes
+
+
+def test_parse_entry_resolves_relative_includes() -> None:
+    """Relative includes are resolved against the entry's ``directory``."""
+    directory = f"{ABS}build/proj"
+    entry = _entry(
+        directory,
+        f"{directory}/src/esphome/x.cpp",
+        "g++ -Iconfig -I../shared -isystem rel/sys -c x.cpp",
+    )
+
+    _, _, includes, _ = idedata._parse_entry(entry)
+
+    def resolved(rel: str) -> str:
+        return os.path.normpath(Path(directory) / rel)
+
+    assert resolved("config") in includes
+    assert resolved("../shared") in includes  # ../ normalized away
+    assert resolved("rel/sys") in includes
+    # nothing is left relative
+    assert all(Path(inc).is_absolute() for inc in includes)
+
+
+def test_parse_entry_skips_dependency_flags() -> None:
+    """Dependency-generation flags (and their args) are dropped."""
+    entry = _entry(
+        "/build",
+        "/build/src/esphome/x.cpp",
+        "g++ -MD -MT x.cpp.o -MF x.cpp.o.d -c x.cpp -o x.cpp.o",
+    )
+
+    _, _, _, cxx_flags = idedata._parse_entry(entry)
+
+    for tok in ("-MD", "-MT", "x.cpp.o", "-MF", "x.cpp.o.d", "-c", "-o", "x.cpp"):
+        assert tok not in cxx_flags
+
+
+def test_expand_response_files(tmp_path: Path) -> None:
+    """``@file`` arguments are inlined relative to the directory."""
+    rsp = tmp_path / "flags.rsp"
+    rsp.write_text("-DFROM_RSP -I/rsp/inc")
+
+    tokens = idedata._expand_response_files(
+        ["g++", f"@{rsp.name}", "-c", "x.cpp"], tmp_path
+    )
+
+    assert "-DFROM_RSP" in tokens
+    assert "-I/rsp/inc" in tokens
+    assert not any(t.startswith("@") for t in tokens)
+
+
+def test_expand_response_files_keeps_literal_when_missing(tmp_path: Path) -> None:
+    """An unreadable ``@file`` token is kept verbatim rather than dropped."""
+    tokens = idedata._expand_response_files(["g++", "@nope.rsp"], tmp_path)
+    assert "@nope.rsp" in tokens
+
+
+def test_pick_entry_prefers_esphome_tu() -> None:
+    """A ``/src/esphome/`` C++ TU is picked over other compile entries."""
+    entries = [
+        _entry("/b", "/b/managed_components/foo/foo.c", "gcc -c foo.c"),
+        _entry("/b", "/b/src/esphome/core/app.cpp", "g++ -c app.cpp"),
+    ]
+    assert idedata._pick_entry(entries)["file"].endswith("app.cpp")
+
+
+def test_idedata_from_build(tmp_path: Path) -> None:
+    """Full transform: representative entry + include union + toolchain dirs."""
+    compile_commands = tmp_path / "compile_commands.json"
+    entries = [
+        _entry(
+            f"{ABS}b",
+            f"{ABS}b/src/esphome/core/app.cpp",
+            f"g++ -DUSE_ESP32 -I{ABS}inc/core -std=gnu++20 -c app.cpp -o app.cpp.o",
+        ),
+        _entry(
+            f"{ABS}b",
+            f"{ABS}b/src/esphome/sensor/s.cpp",
+            f"g++ -DUSE_ESP32 -I{ABS}inc/sensor -c s.cpp -o s.cpp.o",
+        ),
+        # non-esphome TU: its includes must not leak into the union
+        _entry(
+            f"{ABS}b",
+            f"{ABS}b/managed_components/x/x.c",
+            f"gcc -I{ABS}inc/managed -c x.c",
+        ),
+    ]
+    compile_commands.write_text(json.dumps(entries))
+
+    fake_proc = MagicMock(
+        returncode=0,
+        stderr=(
+            "ignored\n"
+            "#include <...> search starts here:\n"
+            " /tc/inc/c++\n"
+            " /tc/inc\n"
+            "End of search list.\n"
+            "more ignored\n"
+        ),
+    )
+    with patch.object(idedata.subprocess, "run", return_value=fake_proc):
+        data = idedata.idedata_from_build(compile_commands)
+
+    assert data["cxx_path"] == "g++"
+    assert "USE_ESP32" in data["defines"]
+    assert "-std=gnu++20" in data["cxx_flags"]
+    # include dirs unioned across all esphome TUs
+    assert f"{ABS}inc/core" in data["includes"]["build"]
+    assert f"{ABS}inc/sensor" in data["includes"]["build"]
+    # the non-esphome TU is excluded from the union
+    assert f"{ABS}inc/managed" not in data["includes"]["build"]
+    # toolchain search dirs parsed from the compiler's -v output
+    assert data["includes"]["toolchain"] == ["/tc/inc/c++", "/tc/inc"]
+
+
+def test_get_toolchain_includes_raises_on_probe_failure() -> None:
+    """A failed compiler probe is a hard error, not a silent empty list."""
+    fake_proc = MagicMock(returncode=1, stderr="xtensa-esp32-elf-g++: not found")
+    with (
+        patch.object(idedata.subprocess, "run", return_value=fake_proc),
+        pytest.raises(RuntimeError, match="builtin include dirs"),
+    ):
+        idedata._get_toolchain_includes("/bad/compiler")
+
+
+def test_get_toolchain_includes_raises_when_no_dirs_found() -> None:
+    """Markers present but no dirs (anomalous output) also raises."""
+    fake_proc = MagicMock(
+        returncode=0,
+        stderr="#include <...> search starts here:\nEnd of search list.\n",
+    )
+    with (
+        patch.object(idedata.subprocess, "run", return_value=fake_proc),
+        pytest.raises(RuntimeError, match="builtin include dirs"),
+    ):
+        idedata._get_toolchain_includes("/some/compiler")

tests/unit_tests/test_espidf_toolchain.py

@@ -2,6 +2,9 @@
 
 # pylint: disable=protected-access
 
+import json
+import os
+from pathlib import Path
 from unittest.mock import patch
 
 from esphome.const import CONF_FRAMEWORK, CONF_SOURCE
@@ -56,3 +59,92 @@ def test_get_esphome_esp_idf_paths_no_override():
     ) as mock_install:
         toolchain._get_esphome_esp_idf_paths("5.5.4")
     mock_install.assert_called_once_with("5.5.4", source_url=None)
+
+
+def _setup_build(setup_core: Path) -> tuple[Path, Path]:
+    """Point CORE at a build dir; return (compile_commands, idedata cache) paths."""
+    CORE.name = "test"
+    CORE.build_path = setup_core / "build" / "test"
+    compile_commands = CORE.relative_build_path("build", "compile_commands.json")
+    cache = CORE.relative_internal_path("idedata", "test.json")
+    return compile_commands, cache
+
+
+def test_get_idedata_returns_none_without_compile_commands(setup_core: Path) -> None:
+    """No compile DB yet -> None (rather than an error)."""
+    _setup_build(setup_core)
+    assert toolchain.get_idedata() is None
+
+
+def test_get_idedata_generates_and_caches(setup_core: Path) -> None:
+    """Generates from the compile DB and writes the cache."""
+    compile_commands, cache = _setup_build(setup_core)
+    compile_commands.parent.mkdir(parents=True, exist_ok=True)
+    compile_commands.write_text("[]")
+
+    with patch(
+        "esphome.espidf.idedata.idedata_from_build",
+        return_value={"cxx_path": "g++"},
+    ) as mock_transform:
+        result = toolchain.get_idedata()
+
+    mock_transform.assert_called_once()
+    assert result == {"cxx_path": "g++"}
+    assert json.loads(cache.read_text()) == {"cxx_path": "g++"}
+
+
+def test_get_idedata_uses_cache_when_valid(setup_core: Path) -> None:
+    """A cache at least as new as the compile DB is reused without regenerating."""
+    compile_commands, cache = _setup_build(setup_core)
+    compile_commands.parent.mkdir(parents=True, exist_ok=True)
+    compile_commands.write_text("[]")
+    cache.parent.mkdir(parents=True, exist_ok=True)
+    cache.write_text('{"cxx_path": "cached"}')
+    cc_mtime = compile_commands.stat().st_mtime
+    os.utime(cache, (cc_mtime + 1, cc_mtime + 1))
+
+    with patch("esphome.espidf.idedata.idedata_from_build") as mock_transform:
+        result = toolchain.get_idedata()
+
+    mock_transform.assert_not_called()
+    assert result == {"cxx_path": "cached"}
+
+
+def test_get_idedata_regenerates_when_compile_commands_newer(setup_core: Path) -> None:
+    """A compile DB newer than the cache forces regeneration."""
+    compile_commands, cache = _setup_build(setup_core)
+    cache.parent.mkdir(parents=True, exist_ok=True)
+    cache.write_text('{"cxx_path": "stale"}')
+    compile_commands.parent.mkdir(parents=True, exist_ok=True)
+    compile_commands.write_text("[]")
+    cache_mtime = cache.stat().st_mtime
+    os.utime(compile_commands, (cache_mtime + 1, cache_mtime + 1))
+
+    with patch(
+        "esphome.espidf.idedata.idedata_from_build",
+        return_value={"cxx_path": "fresh"},
+    ) as mock_transform:
+        result = toolchain.get_idedata()
+
+    mock_transform.assert_called_once()
+    assert result == {"cxx_path": "fresh"}
+
+
+def test_get_idedata_regenerates_on_corrupted_cache(setup_core: Path) -> None:
+    """An unparseable (but newer) cache falls back to regeneration."""
+    compile_commands, cache = _setup_build(setup_core)
+    compile_commands.parent.mkdir(parents=True, exist_ok=True)
+    compile_commands.write_text("[]")
+    cache.parent.mkdir(parents=True, exist_ok=True)
+    cache.write_text("{not json")
+    cc_mtime = compile_commands.stat().st_mtime
+    os.utime(cache, (cc_mtime + 1, cc_mtime + 1))
+
+    with patch(
+        "esphome.espidf.idedata.idedata_from_build",
+        return_value={"cxx_path": "regen"},
+    ) as mock_transform:
+        result = toolchain.get_idedata()
+
+    mock_transform.assert_called_once()
+    assert result == {"cxx_path": "regen"}