Wrap ipv6 code a bit more (#4574)

* Wrap ipv6 code a bit more for when ipv6 support should not be compiled in

* More checks

* More uses

* Fix

esphome/components/api/api_server.cpp

@@ -45,7 +45,7 @@ void APIServer::setup() {
 
   struct sockaddr_storage server;
 
-  socklen_t sl = socket::set_sockaddr_any((struct sockaddr *) &server, sizeof(server), htons(this->port_));
+  socklen_t sl = socket::set_sockaddr_any((struct sockaddr *) &server, sizeof(server), this->port_);
   if (sl == 0) {
     ESP_LOGW(TAG, "Socket unable to set sockaddr: errno %d", errno);
     this->mark_failed();