[packages] Resolve git symlinks on Windows when materialized as text (#16657)

esphome/components/packages/__init__.py

@@ -215,7 +215,7 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]:
     If loading fails after cloning, attempts a revert and retry in case
     a prior cached checkout is stale.
     """
-    repo_dir, revert = git.clone_or_update(
+    repo_root, revert = git.clone_or_update(
         url=config[CONF_URL],
         ref=config.get(CONF_REF),
         refresh=config[CONF_REFRESH],
@@ -225,6 +225,10 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]:
     )
     files: list[dict[str, Any]] = []
 
+    # ``repo_root`` is the directory containing ``.git`` and must be passed
+    # to git for symlink-stub resolution. ``repo_dir`` may be narrowed to a
+    # subdirectory via the user's CONF_PATH and is used for file lookups.
+    repo_dir = repo_root
     if base_path := config.get(CONF_PATH):
         repo_dir = repo_dir / base_path
 
@@ -236,13 +240,37 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]:
 
     def _load_package_yaml(yaml_file: Path, filename: str) -> dict:
         """Load a YAML file from a remote package, validating min_version."""
-        try:
+
-            new_yaml = yaml_util.load_yaml(yaml_file)
+        def _load(path: Path) -> dict | str | None:
-        except EsphomeError as e:
+            try:
+                return yaml_util.load_yaml(path)
+            except EsphomeError as e:
+                raise cv.Invalid(
+                    f"{filename} is not a valid YAML file."
+                    f" Please check the file contents.\n{e}"
+                ) from e
+
+        new_yaml = _load(yaml_file)
+        if not isinstance(new_yaml, dict):
+            # On Windows, git defaults to core.symlinks=false unless the user
+            # has Developer Mode enabled or is running elevated. Files stored
+            # in the repo as symlinks (tree mode 120000) are then checked out
+            # as plain text files containing the symlink target path, so
+            # parsing them as YAML yields a bare scalar instead of a mapping.
+            # Best-effort: follow the symlink target ourselves and re-load.
+            target = git.resolve_symlink_stub(repo_root, yaml_file)
+            if target is not None:
+                new_yaml = _load(target)
+        if not isinstance(new_yaml, dict):
             raise cv.Invalid(
-                f"{filename} is not a valid YAML file."
+                f"{filename} does not contain a YAML mapping at the top level "
-                f" Please check the file contents.\n{e}"
+                f"(got {type(new_yaml).__name__}). "
-            ) from e
+                f"If this file is a git symlink in the source repository, it "
+                f"may not have been materialized correctly on your platform "
+                f"(this is a known issue with git on Windows without Developer "
+                f"Mode enabled). Try pointing your package at the real file "
+                f"path instead."
+            )
         esphome_config = new_yaml.get(CONF_ESPHOME) or {}
         min_version = esphome_config.get(CONF_MIN_VERSION)
         if min_version is not None and cv.Version.parse(min_version) > cv.Version.parse(

esphome/git.py

@@ -6,6 +6,7 @@ import logging
 from pathlib import Path
 import re
 import subprocess
+import sys
 import urllib.parse
 
 import esphome.config_validation as cv
@@ -93,6 +94,92 @@ def _compute_destination_path(key: str, domain: str) -> Path:
     return base_dir / h.hexdigest()[:8]
 
 
+def resolve_symlink_stub(repo_dir: Path, file_path: Path) -> Path | None:
+    """Return the symlink target if ``file_path`` is a Windows-checked-out symlink stub.
+
+    On Windows, when ``core.symlinks=false`` (the default unless the user has
+    SeCreateSymbolicLinkPrivilege — i.e. Developer Mode or running elevated),
+    git materializes files with tree mode ``120000`` as plain text files
+    whose content is the literal symlink target path. Opening such a file
+    yields the target path string instead of the target's content.
+
+    If ``file_path`` is one of those stubs, return the resolved target Path
+    inside ``repo_dir``. Otherwise return ``None`` and the caller should use
+    ``file_path`` as-is.
+
+    Designed to be called *only* when normal access has already produced an
+    unexpected result (e.g. YAML parsed as a top-level scalar), so the
+    per-file ``git ls-files`` subprocess cost is paid only on the failure
+    path. Returns ``None`` on any error or check failure — it's purely a
+    best-effort recovery, never raises.
+    """
+    # On non-Windows, git creates real symlinks; ordinary file access already
+    # transparently follows them.
+    if sys.platform != "win32":
+        return None
+    if file_path.is_symlink():
+        return None
+    if not file_path.is_file():
+        return None
+
+    try:
+        rel = file_path.relative_to(repo_dir)
+    except ValueError:
+        return None
+
+    try:
+        # ``git ls-files -s <path>`` prints "<mode> <sha> <stage>\t<path>"
+        # for that single entry, or empty if untracked.
+        out = run_git_command(
+            ["git", "ls-files", "-s", "--", rel.as_posix()],
+            git_dir=repo_dir,
+        )
+    except GitException:
+        return None
+
+    parts = out.split()
+    if not parts or parts[0] != "120000":
+        return None
+
+    # Stubs are short ASCII relative paths. Decode defensively, and only
+    # strip the trailing newline git's checkout may append — preserving any
+    # whitespace that could be part of a valid target name.
+    try:
+        raw = file_path.read_bytes()
+    except OSError:
+        return None
+    try:
+        target_str = raw.decode("utf-8").rstrip("\r\n")
+    except UnicodeDecodeError:
+        return None
+
+    # ``Path()`` and ``Path.resolve()`` can raise on malformed inputs (e.g.
+    # embedded NUL bytes from a hostile symlink blob, paths too long for the
+    # OS, or temporary I/O errors). Catch broadly — this helper is purely a
+    # best-effort recovery and must never raise.
+    try:
+        target_path = (file_path.parent / target_str).resolve()
+        repo_root_resolved = repo_dir.resolve()
+    except (OSError, ValueError, RuntimeError):
+        return None
+
+    # ``Path.resolve()`` follows ``..``; re-verify containment afterwards.
+    try:
+        target_path.relative_to(repo_root_resolved)
+    except ValueError:
+        _LOGGER.warning(
+            "Refusing to follow symlink %s -> %s (escapes repository)",
+            file_path,
+            target_str,
+        )
+        return None
+
+    if not target_path.is_file():
+        return None
+
+    return target_path
+
+
 def clone_or_update(
     *,
     url: str,

tests/unit_tests/test_git.py

@@ -4,7 +4,7 @@ from datetime import datetime, timedelta
 import os
 from pathlib import Path
 from typing import Any
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
 
 import pytest
 
@@ -1001,3 +1001,304 @@ def test_refresh_picks_up_new_remote_commits(
         "--hard",
         "old_sha",
     ]
+
+
+def test_resolve_symlink_stub_returns_none_on_non_windows(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """On non-Windows, resolve_symlink_stub returns None without calling git."""
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+    stub = repo_dir / "file.yaml"
+    stub.write_text("static/file.yaml")
+
+    with patch("esphome.git.sys.platform", "linux"):
+        result = git.resolve_symlink_stub(repo_dir, stub)
+
+    assert result is None
+    mock_run_git_command.assert_not_called()
+
+
+def test_resolve_symlink_stub_returns_target_for_mode_120000(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """A mode-120000 file is recognised as a stub; its target Path is returned."""
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+    (repo_dir / "static").mkdir()
+
+    target = repo_dir / "static" / "real.yaml"
+    target.write_text("esphome:\n  name: real\n")
+
+    stub = repo_dir / "real.yaml"
+    stub.write_text("static/real.yaml")
+
+    mock_run_git_command.return_value = "120000 abc123 0\treal.yaml"
+
+    with patch("esphome.git.sys.platform", "win32"):
+        result = git.resolve_symlink_stub(repo_dir, stub)
+
+    assert result == target.resolve()
+    # Stub file itself was not modified — only inspected.
+    assert stub.read_text() == "static/real.yaml"
+
+
+def test_resolve_symlink_stub_resolves_relative_parent_paths(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """Symlink targets with ``..`` segments resolve correctly within the repo."""
+    repo_dir = tmp_path / "repo"
+    (repo_dir / "subdir").mkdir(parents=True)
+    (repo_dir / "static").mkdir()
+
+    target = repo_dir / "static" / "shared.yaml"
+    target.write_text("shared content")
+
+    stub = repo_dir / "subdir" / "shared.yaml"
+    stub.write_text("../static/shared.yaml")
+
+    mock_run_git_command.return_value = "120000 abc123 0\tsubdir/shared.yaml"
+
+    with patch("esphome.git.sys.platform", "win32"):
+        result = git.resolve_symlink_stub(repo_dir, stub)
+
+    assert result == target.resolve()
+
+
+def test_resolve_symlink_stub_refuses_escape_outside_repo(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """A symlink pointing outside the repository is not followed."""
+    outside = tmp_path / "outside.yaml"
+    outside.write_text("sensitive")
+
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+
+    stub = repo_dir / "escape.yaml"
+    stub.write_text("../outside.yaml")
+
+    mock_run_git_command.return_value = "120000 abc123 0\tescape.yaml"
+
+    with patch("esphome.git.sys.platform", "win32"):
+        result = git.resolve_symlink_stub(repo_dir, stub)
+
+    assert result is None
+
+
+def test_resolve_symlink_stub_returns_none_for_real_symlink(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """A real symlink already opens transparently, so the helper short-circuits.
+
+    Skipped on Windows where symlink creation requires
+    SeCreateSymbolicLinkPrivilege.
+    """
+    if os.name == "nt":
+        pytest.skip("Requires symlink-creation privilege on Windows")
+
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+    target = repo_dir / "real.yaml"
+    target.write_text("real content")
+
+    real_link = repo_dir / "link.yaml"
+    real_link.symlink_to("real.yaml")
+
+    with patch("esphome.git.sys.platform", "win32"):
+        result = git.resolve_symlink_stub(repo_dir, real_link)
+
+    assert result is None
+    # No git call needed for real symlinks.
+    mock_run_git_command.assert_not_called()
+
+
+def test_resolve_symlink_stub_returns_none_for_regular_file(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """A regular file (mode 100644) whose content looks path-shaped is not
+    followed."""
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+
+    regular = repo_dir / "looks_like_path.txt"
+    regular.write_text("static/something.yaml")
+
+    mock_run_git_command.return_value = "100644 abc123 0\tlooks_like_path.txt"
+
+    with patch("esphome.git.sys.platform", "win32"):
+        result = git.resolve_symlink_stub(repo_dir, regular)
+
+    assert result is None
+
+
+def test_resolve_symlink_stub_returns_none_when_git_fails(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """If ``git ls-files`` fails (e.g. not a repo), the helper returns None."""
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+
+    stub = repo_dir / "real.yaml"
+    stub.write_text("static/real.yaml")
+
+    mock_run_git_command.side_effect = GitCommandError("ls-files exploded")
+
+    with patch("esphome.git.sys.platform", "win32"):
+        result = git.resolve_symlink_stub(repo_dir, stub)
+
+    assert result is None
+
+
+def test_resolve_symlink_stub_returns_none_for_non_utf8_content(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """A file whose bytes are not valid UTF-8 must not raise — return None."""
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+
+    stub = repo_dir / "binary.bin"
+    stub.write_bytes(b"\xff\xfe\x00\xff")
+
+    mock_run_git_command.return_value = "120000 abc123 0\tbinary.bin"
+
+    with patch("esphome.git.sys.platform", "win32"):
+        result = git.resolve_symlink_stub(repo_dir, stub)
+
+    assert result is None
+
+
+def test_resolve_symlink_stub_preserves_whitespace_in_target(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """Only trailing CR/LF is stripped — internal whitespace is preserved."""
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+    target_dir = repo_dir / "dir with spaces"
+    target_dir.mkdir()
+    target = target_dir / "real.yaml"
+    target.write_text("hello")
+
+    stub = repo_dir / "link.yaml"
+    # Trailing newline (as git's checkout may append) is stripped, but
+    # whitespace inside the target path itself must survive.
+    stub.write_bytes(b"dir with spaces/real.yaml\n")
+
+    mock_run_git_command.return_value = "120000 abc123 0\tlink.yaml"
+
+    with patch("esphome.git.sys.platform", "win32"):
+        result = git.resolve_symlink_stub(repo_dir, stub)
+
+    assert result == target.resolve()
+
+
+def test_resolve_symlink_stub_returns_none_for_directory_target(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """A symlink pointing at a directory has no file content to load."""
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+    (repo_dir / "dir_target").mkdir()
+
+    stub = repo_dir / "link_to_dir"
+    stub.write_text("dir_target")
+
+    mock_run_git_command.return_value = "120000 abc123 0\tlink_to_dir"
+
+    with patch("esphome.git.sys.platform", "win32"):
+        result = git.resolve_symlink_stub(repo_dir, stub)
+
+    assert result is None
+
+
+def test_resolve_symlink_stub_returns_none_when_resolve_raises(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """Path.resolve() raising (e.g. on a malformed target) must not propagate."""
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+
+    stub = repo_dir / "broken.yaml"
+    stub.write_text("ignored")
+
+    mock_run_git_command.return_value = "120000 abc123 0\tbroken.yaml"
+
+    with (
+        patch("esphome.git.sys.platform", "win32"),
+        patch.object(Path, "resolve", side_effect=OSError("bad path")),
+    ):
+        result = git.resolve_symlink_stub(repo_dir, stub)
+
+    assert result is None
+
+
+def test_resolve_symlink_stub_returns_none_when_file_missing(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """A file path that doesn't exist is rejected before git is consulted."""
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+
+    missing = repo_dir / "ghost.yaml"  # not created
+
+    with patch("esphome.git.sys.platform", "win32"):
+        result = git.resolve_symlink_stub(repo_dir, missing)
+
+    assert result is None
+    mock_run_git_command.assert_not_called()
+
+
+def test_resolve_symlink_stub_returns_none_when_path_outside_repo(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """A file path that isn't under repo_dir is rejected (ValueError from relative_to)."""
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+
+    outside = tmp_path / "stray.yaml"
+    outside.write_text("something")
+
+    with patch("esphome.git.sys.platform", "win32"):
+        result = git.resolve_symlink_stub(repo_dir, outside)
+
+    assert result is None
+    mock_run_git_command.assert_not_called()
+
+
+def test_resolve_symlink_stub_returns_none_when_untracked(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """Empty `git ls-files` output (untracked file) makes the helper return None."""
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+
+    stub = repo_dir / "untracked.yaml"
+    stub.write_text("static/foo.yaml")
+
+    mock_run_git_command.return_value = ""
+
+    with patch("esphome.git.sys.platform", "win32"):
+        result = git.resolve_symlink_stub(repo_dir, stub)
+
+    assert result is None
+
+
+def test_resolve_symlink_stub_returns_none_when_read_bytes_raises(
+    tmp_path: Path, mock_run_git_command: Mock
+) -> None:
+    """An OSError from read_bytes() (e.g. file vanished mid-call) must not propagate."""
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+
+    stub = repo_dir / "racy.yaml"
+    stub.write_text("static/racy.yaml")
+
+    mock_run_git_command.return_value = "120000 abc123 0\tracy.yaml"
+
+    with (
+        patch("esphome.git.sys.platform", "win32"),
+        patch.object(Path, "read_bytes", side_effect=OSError("vanished")),
+    ):
+        result = git.resolve_symlink_stub(repo_dir, stub)
+
+    assert result is None

tests/unit_tests/test_substitutions.py

@@ -838,3 +838,86 @@ def test_include_vars_applied_to_lambda_value(tmp_path: Path) -> None:
 
     assert isinstance(result["value"], Lambda)
     assert result["value"].value == 'return "bar";'
+
+
+@patch("esphome.git.resolve_symlink_stub")
+@patch("esphome.git.clone_or_update")
+def test_remote_package_symlink_stub_is_followed(
+    mock_clone_or_update: MagicMock,
+    mock_resolve_symlink_stub: MagicMock,
+    tmp_path: Path,
+) -> None:
+    """When a package YAML is a scalar (symlink stub) and resolve_symlink_stub
+    returns a target, the loader follows the target and uses its content."""
+    CORE.config_path = tmp_path / "test.yaml"
+
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+    (repo_dir / "static").mkdir()
+
+    # Stub file: content is the target path string (simulating Windows behavior).
+    stub = repo_dir / "file1.yaml"
+    stub.write_text("static/file1.yaml")
+
+    # Real target with valid YAML mapping.
+    target = repo_dir / "static" / "file1.yaml"
+    target.write_text("substitutions:\n  hello: world\n")
+
+    mock_clone_or_update.return_value = (repo_dir, None)
+    mock_resolve_symlink_stub.return_value = target
+
+    config: dict[str, Any] = {
+        "packages": {
+            "test_package": {
+                "url": "https://github.com/esphome/repo1",
+                "ref": "main",
+                "files": ["file1.yaml"],
+            }
+        }
+    }
+
+    # Must succeed (does not raise the helpful cv.Invalid) because the stub
+    # was followed and a valid mapping was loaded from the target.
+    do_packages_pass(config)
+    assert mock_resolve_symlink_stub.called
+
+
+@patch("esphome.git.clone_or_update")
+def test_remote_package_scalar_yaml_raises_helpful_error(
+    mock_clone_or_update: MagicMock, tmp_path: Path
+) -> None:
+    """A remote package YAML that is a top-level scalar (e.g. an unmaterialized
+    git symlink on Windows) raises a clear cv.Invalid, not AttributeError.
+
+    Regression test for the case where a repo containing a YAML symlink,
+    checked out on Windows without symlink privilege, lands as a short text
+    file containing the symlink target path. PyYAML parses that as a bare
+    string scalar; the package loader must reject it with a human-readable
+    error instead of dying inside ``.get()``.
+    """
+    CORE.config_path = tmp_path / "test.yaml"
+
+    repo_dir = tmp_path / "repo"
+    repo_dir.mkdir()
+    # Simulate the broken-symlink state: a YAML file whose entire content is
+    # the symlink target string. PyYAML parses this as a top-level scalar.
+    (repo_dir / "file1.yaml").write_text("static/file1.yaml")
+
+    mock_clone_or_update.return_value = (repo_dir, None)
+
+    config: dict[str, Any] = {
+        "packages": {
+            "test_package": {
+                "url": "https://github.com/esphome/repo1",
+                "ref": "main",
+                "files": ["file1.yaml"],
+            }
+        }
+    }
+
+    with pytest.raises(cv.Invalid) as exc_info:
+        do_packages_pass(config)
+
+    msg = str(exc_info.value)
+    assert "mapping at the top level" in msg
+    assert "file1.yaml" in msg