diff --git a/common b/common index c442a56df4e..e82f88c90a6 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit c442a56df4e227eeebb3ae4015dd2c741d7af389 +Subproject commit e82f88c90a6324701435680d77b538f5bc057058 diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 55d3699982f..068c9315120 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -32,6 +32,9 @@ services: environment: - NGINX_WEB_SERVER_NAME=${NGINX_WEB_SERVER_NAME:-devthedocs.org} - NGINX_PROXITO_SERVER_NAME=${NGINX_PROXITO_SERVER_NAME:-*.devthedocs.org *.build.devthedocs.org} + # The other environment, see `common/dockerfiles/nginx/side-by-side.conf.template` + - NGINX_SIDE_BY_SIDE_SERVER_NAME=${NGINX_SIDE_BY_SIDE_SERVER_NAME:-.devthedocs.com} + - NGINX_SIDE_BY_SIDE_PORT=${NGINX_SIDE_BY_SIDE_PORT:-10081} proxito: image: community-server:latest diff --git a/dockerfiles/nginx/README.rst b/dockerfiles/nginx/README.rst index 91b2d17550b..e5a73cedeab 100644 --- a/dockerfiles/nginx/README.rst +++ b/dockerfiles/nginx/README.rst @@ -1,7 +1,7 @@ How NGINX proxy works ===================== -Read the Docs uses 3 different NGINX configurations; +Read the Docs uses 5 different NGINX configurations; web This configuration is in charge of serving the dashboard application @@ -42,6 +42,17 @@ wrangler on port ``8080`` to fetch the "original response" and manipulates it to inject extra HTTP tags required for Read the Docs Addons (``meta`` and ``script``). +services + Serves the development services by hostname, so they don't need host ports: + ``assets.``, ``addons.``, ``storage.``, ``storage-console.`` and ``search.$NGINX_WEB_SERVER_NAME``. + Shared with business, from ``common/dockerfiles/nginx/``. + +side-by-side + Forwards the other environment's domains (``$NGINX_SIDE_BY_SIDE_SERVER_NAME``) + to its NGINX on host port ``$NGINX_SIDE_BY_SIDE_PORT``, + so community and business can run at the same time. + Shared with business, from ``common/dockerfiles/nginx/``. + ASCII-art explanation diff --git a/dockerfiles/nginx/services.conf.template b/dockerfiles/nginx/services.conf.template new file mode 120000 index 00000000000..39a38d1f791 --- /dev/null +++ b/dockerfiles/nginx/services.conf.template @@ -0,0 +1 @@ +../../common/dockerfiles/nginx/services.conf.template \ No newline at end of file diff --git a/dockerfiles/nginx/side-by-side.conf.template b/dockerfiles/nginx/side-by-side.conf.template new file mode 120000 index 00000000000..cd756102dbe --- /dev/null +++ b/dockerfiles/nginx/side-by-side.conf.template @@ -0,0 +1 @@ +../../common/dockerfiles/nginx/side-by-side.conf.template \ No newline at end of file diff --git a/docs/dev/install.rst b/docs/dev/install.rst index 3c5118790fe..b139f1cbd73 100644 --- a/docs/dev/install.rst +++ b/docs/dev/install.rst @@ -225,6 +225,48 @@ save some work while typing docker compose commands. This section explains these Pre-compile and cache tools that can be specified in ``build.tools`` to speed up builds. It requires ``inv docker.up`` running in another terminal to be able to upload the pre-compiled version to the cache. +Development services +~~~~~~~~~~~~~~~~~~~~ + +Only NGINX is published on the host. +It serves the other development services by hostname, on the same port as the dashboard +(``devthedocs.org`` for community, ``devthedocs.com`` for business): + +.. list-table:: + :header-rows: 1 + + * - Hostname + - Service + * - ``assets.devthedocs.org`` + - ext-theme assets development server (webpack) + * - ``addons.devthedocs.org`` + - Addons development server + * - ``storage.devthedocs.org`` + - S3 storage API, used by the upload API's presigned URLs + * - ``storage-console.devthedocs.org`` + - S3 storage web console + * - ``search.devthedocs.org`` + - Elasticsearch + +Running community and business side by side +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Both ``devthedocs.org`` and ``devthedocs.com`` resolve to ``127.0.0.1``, +so the two NGINX containers can't both own port 80. +Instead, the environment that owns port 80 forwards the other one's domains to its NGINX, +so all the URLs stay the same. + +Start both with ``inv docker.up``, in any order. +``inv docker.up`` publishes NGINX on port 80 unless another container already does, +in which case it uses the side-by-side port: ``10080`` for community, ``10081`` for business. +Set ``RTDDEV_PORT_NGINX`` to choose the port yourself. + +.. note:: + + Running both environments takes around 11 GB of memory. + Give Docker at least 16 GB, otherwise containers get killed when it runs out of memory. + The first one to go is usually a webpack container, which leaves the dashboard without styles. + Adding a new Python dependency ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 6960b4f171c..b04f32d08f4 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -469,7 +469,9 @@ def CSP_CONNECT_SRC(self): # Plausible analytics "https://plausible.io/api/event", ] - CSP_CONNECT_SRC.append(f"ws://{self.PRODUCTION_DOMAIN}:10001/ws") + if self.RTD_EXT_THEME_DEV_SERVER: + # Live reload websocket of the webpack development server. + CSP_CONNECT_SRC.append(self.RTD_EXT_THEME_DEV_SERVER.replace("http", "ws", 1) + "/ws") return CSP_CONNECT_SRC @property diff --git a/readthedocs/settings/docker_compose.py b/readthedocs/settings/docker_compose.py index 44e56ad73bd..c0deaaa4905 100644 --- a/readthedocs/settings/docker_compose.py +++ b/readthedocs/settings/docker_compose.py @@ -88,7 +88,7 @@ def DOCROOT(self): @property def RTD_EXT_THEME_DEV_SERVER(self): if self.RTD_EXT_THEME_DEV_SERVER_ENABLED: - return "http://assets.devthedocs.org:10001" + return f"http://assets.{self.PRODUCTION_DOMAIN}" # Enable auto syncing elasticsearch documents ELASTICSEARCH_DSL_AUTOSYNC = "SEARCH" in os.environ diff --git a/readthedocs/upload/api/views.py b/readthedocs/upload/api/views.py index 1d197b81054..8c038393ea4 100644 --- a/readthedocs/upload/api/views.py +++ b/readthedocs/upload/api/views.py @@ -210,8 +210,13 @@ def _generate_upload_url(self, build): ) if settings.RTD_DOCKER_COMPOSE and not settings.USING_AWS: # Overriden so we return the public URL for uploading artifacts, - # instead of the internal hostname (http://storage), which is not accessible from the host machine. - response["url"] = response["url"].replace("://storage", "://127.0.0.1", 1) + # instead of the internal hostname (http://storage:9000), which is not accessible from the host machine. + # NGINX serves the storage container on this hostname. + response["url"] = response["url"].replace( + "://storage:9000", + f"://storage.{settings.PRODUCTION_DOMAIN}", + 1, + ) return response diff --git a/readthedocs/upload/tests/test_views.py b/readthedocs/upload/tests/test_views.py index be84b722c2c..87e3928b4fb 100644 --- a/readthedocs/upload/tests/test_views.py +++ b/readthedocs/upload/tests/test_views.py @@ -310,18 +310,22 @@ def test_cancels_running_builds_for_same_version( ) assert running_build.pk != response.data["build"]["id"] - @override_settings(RTD_DOCKER_COMPOSE=True, USING_AWS=False) + @override_settings( + RTD_DOCKER_COMPOSE=True, + USING_AWS=False, + PRODUCTION_DOMAIN="devthedocs.com", + ) @mock.patch("readthedocs.projects.tasks.utils.send_build_status") @mock.patch("readthedocs.upload.api.views.storages") def test_docker_compose_replaces_storage_hostname(self, storages_mock, send_build_status): storage_mock = self._mock_storage(storages_mock) storage_mock.generate_presigned_post.return_value = { - "url": "http://storage/build-uploads", + "url": "http://storage:9000/build-uploads", "fields": {"key": "project/1/artifacts.zip"}, } response = self.client.post(self.url, self.data) assert response.status_code == status.HTTP_201_CREATED - assert response.data["upload_url"]["url"] == "http://127.0.0.1/build-uploads" + assert response.data["upload_url"]["url"] == "http://storage.devthedocs.com/build-uploads" @mock.patch("readthedocs.projects.tasks.utils.send_build_status") @mock.patch("readthedocs.upload.api.views.storages")