[esp32] Clear IDF environment variables (#10527)

Co-authored-by: J. Nick Koston <nick+github@koston.org>

esphome/components/esp32/__init__.py

@@ -841,6 +841,9 @@ async def to_code(config):
     if conf[CONF_ADVANCED][CONF_IGNORE_EFUSE_CUSTOM_MAC]:
         cg.add_define("USE_ESP32_IGNORE_EFUSE_CUSTOM_MAC")
 
+    for clean_var in ("IDF_PATH", "IDF_TOOLS_PATH"):
+        os.environ.pop(clean_var, None)
+
     add_extra_script(
         "post",
         "post_build.py",