Tests: a WebUI server whose port was taken starts again on another

The host tests found a free port by binding port 0 and closing it, then started the
WebUI's server on it. In between, any connection on the machine can take that port as
its own (the same ephemeral range): the server then exits at once, "Failed to bind to
port N: Address already in use", and the test waited out its window and failed with
"the server did not start" (test_scraper, test_scraper_launchbox) or "HTTP server
exited" (test_webui). It happened in about one gate run in three, under JOBS=4 or not.

tests/webui_build.py start() starts the server, waits for its status and returns the
process, port and session token; a server that exits before answering is started again
on another port (five tries). Every harness that started the server itself uses it:
test_scraper, test_scraper_launchbox, test_scraper_screenscraper, test_webui (its start
and two restarts), test_webui_language, test_webui_transfer_browser. test_scraper's
free_port() had no other user and is gone.

Evidence:
- the cause: 40 sequential starts with each server's stderr kept: 1 failed, "Failed to
  bind to port 44505: Address already in use" (exit 1, after 0.02 s); through start(),
  40 of 40 started.
- python3 -m unittest tests.test_scraper tests.test_scraper_launchbox
  tests.test_scraper_screenscraper: OK; tests.test_webui tests.test_webui_language
  tests.test_webui_transfer_browser: 21 tests OK.
- gates on the tree with the next commit: format, unit (210 tests), build, integration
  (210 tests), evidence - PASS.

tests/test_scraper.py

@@ -87,12 +87,6 @@ class Handler(http.server.BaseHTTPRequestHandler):
         self.answer(False)
 
 
-def free_port():
-    with socket.socket() as s:
-        s.bind(('127.0.0.1', 0))
-        return s.getsockname()[1]
-
-
 class Scraper(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -134,19 +128,10 @@ class Scraper(unittest.TestCase):
         (self.root / f'playlists/{DB}.lpl').write_text(json.dumps({'version': '1.5', 'items': items}))
 
     def start_server(self, lease=None):
-        self.port = free_port()
         env = {**os.environ, 'PS5_SCRAPER_LIBRETRO_BASE': f'http://127.0.0.1:{self.source.server_port}'}
         if lease:
             env['PS5_SCRAPER_LEASE_SECONDS'] = str(lease)
-        self.server = subprocess.Popen([str(self.binary), str(self.root), str(self.port)], env=env,
+        self.server, self.port, self.token = webui_build.start(self.binary, self.root, env)
-                                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        for _ in range(200):
-            try:
-                self.token = json.loads(self.request('GET', '/api/status')[1])['token']
-                return
-            except OSError:
-                time.sleep(0.02)
-        raise RuntimeError('the server did not start')
 
     def stop_server(self):
         self.server.terminate(); self.server.wait(timeout=10)

tests/test_scraper_launchbox.py

@@ -140,18 +140,9 @@ class LaunchBox(unittest.TestCase):
         self.start_server()
 
     def start_server(self):
-        self.port = base.free_port()
         env = {**os.environ, 'PS5_SCRAPER_LAUNCHBOX_BASE': f'http://127.0.0.1:{self.source.server_port}',
                'PS5_SCRAPER_LIBRETRO_BASE': f'http://127.0.0.1:{self.libretro.server_port}'}
-        self.server = subprocess.Popen([str(self.binary), str(self.root), str(self.port)], env=env,
+        self.server, self.port, self.token = webui_build.start(self.binary, self.root, env)
-                                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        for _ in range(200):
-            try:
-                self.token = json.loads(self.request('GET', '/api/status')[1])['token']
-                return
-            except OSError:
-                time.sleep(0.02)
-        raise RuntimeError('the server did not start')
 
     def tearDown(self):
         if self.server.poll() is None:

tests/test_scraper_screenscraper.py

@@ -167,20 +167,11 @@ class ScreenScraper(unittest.TestCase):
         self.start_server()
 
     def start_server(self, developer=True):
-        self.port = base.free_port()
         env = {**os.environ, 'PS5_SCRAPER_SCREENSCRAPER_BASE': f'http://127.0.0.1:{self.source.server_port}',
                'PS5_SCRAPER_LIBRETRO_BASE': f'http://127.0.0.1:{self.libretro.server_port}'}
         if developer:
             env.update(PS5_SCRAPER_TEST_DEVID=DEV_ID, PS5_SCRAPER_TEST_DEVPASSWORD=DEV_PASSWORD)
-        self.server = subprocess.Popen([str(self.binary), str(self.root), str(self.port)], env=env,
+        self.server, self.port, self.token = webui_build.start(self.binary, self.root, env)
-                                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        for _ in range(200):
-            try:
-                self.token = json.loads(self.request('GET', '/api/status')[1])['token']
-                return
-            except OSError:
-                time.sleep(0.02)
-        raise RuntimeError('the server did not start')
 
     def tearDown(self):
         if self.server.poll() is None:

tests/test_webui.py

@@ -30,18 +30,7 @@ class WebUI(unittest.TestCase):
         (cls.root / 'config/retroarch.cfg').write_bytes(cls.original)
         cls.binary = cls.root / 'server'
         webui_build.build(cls.binary, 'tests/webui_server_main.cpp')
-        with socket.socket() as s:
+        cls.process, cls.port, cls.token = webui_build.start(cls.binary, cls.root)
-            s.bind(('127.0.0.1', 0)); cls.port = s.getsockname()[1]
-        cls.process = subprocess.Popen([str(cls.binary), str(cls.root), str(cls.port)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        for _ in range(100):
-            try:
-                with socket.create_connection(('127.0.0.1', cls.port), timeout=.1): break
-            except OSError:
-                if cls.process.poll() is not None: raise RuntimeError('HTTP server exited')
-                time.sleep(.02)
-        else: raise RuntimeError('HTTP server did not start')
-        _, _, body = cls.request('GET', '/api/status')
-        cls.token = json.loads(body)['token']
 
     @classmethod
     def tearDownClass(cls):
@@ -180,14 +169,8 @@ class WebUI(unittest.TestCase):
             self.assertEqual(self.request('POST', endpoint, (key + '=chosen').encode(),
                              {'X-RetroArch-Revision': current['revision']})[0], 200)
         self.process.terminate(); self.process.wait(timeout=5)
-        self.__class__.process = subprocess.Popen([str(self.binary), str(self.root), str(self.port)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        cls = self.__class__
-        for _ in range(100):
+        cls.process, cls.port, cls.token = webui_build.start(self.binary, self.root)
-            try:
-                status, _, body = self.request('GET', '/api/status')
-                if status == 200: break
-            except OSError: pass
-            time.sleep(.02)
-        self.__class__.token = json.loads(body)['token']
         self.assertEqual((self.root / 'config/Fresh Core/Fresh Core.opt').read_text(), 'fresh_resolution = "1"\n')
         self.assertFalse((self.root / 'config/webui-cores/Fresh Core.opt').exists())
         for core in ('PPSSPP', 'dolphin-emu'):
@@ -321,15 +304,7 @@ class WebUI(unittest.TestCase):
         (profile / 'Example Core.opt').write_bytes(original)
         cls = self.__class__
         cls.process.terminate(); cls.process.wait(timeout=5)
-        cls.process = subprocess.Popen([str(cls.binary), str(cls.root), str(cls.port)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        cls.process, cls.port, cls.token = webui_build.start(cls.binary, cls.root)
-        for _ in range(100):
-            try:
-                status, _, body = cls.request('GET', '/api/status')
-                if status == 200: break
-            except OSError: pass
-            time.sleep(.02)
-        else: self.fail('Server did not restart')
-        cls.token = json.loads(body)['token']
         self.assertIn(b'example_resolution = "4x"', (profile / 'Example Core.opt').read_bytes())
         self.assertIn(b'example_filter = "nearest"', (profile / 'Example Core.opt').read_bytes())
         self.assertIn(b'video_vsync = "false"', (profile / 'Example Core.cfg').read_bytes())

tests/test_webui_language.py

@@ -5,10 +5,8 @@ import http.client
 import json
 import os
 import shutil
-import socket
 import subprocess
 import tempfile
-import time
 import unittest
 
 import sys
@@ -43,17 +41,7 @@ class Server:
         subprocess.check_output(['bash', 'tools/build-webui-update.sh', 'host'], cwd=ROOT, text=True)
         self.binary = root.parent / 'server'
         webui_build.build(self.binary, 'tests/webui_server_main.cpp', flags=('-O1',))
-        with socket.socket() as s:
+        self.process, self.port, self.token = webui_build.start(self.binary, root)
-            s.bind(('127.0.0.1', 0)); self.port = s.getsockname()[1]
-        self.process = subprocess.Popen([str(self.binary), str(root), str(self.port)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        for _ in range(100):
-            try:
-                with socket.create_connection(('127.0.0.1', self.port), timeout=.1): break
-            except OSError:
-                if self.process.poll() is not None: raise RuntimeError('HTTP server exited')
-                time.sleep(.02)
-        self.token = ''
-        self.token = self.json('GET', '/api/status')[1]['token']
 
     def request(self, method, path, token=None):
         conn = http.client.HTTPConnection('127.0.0.1', self.port, timeout=5)

tests/test_webui_transfer_browser.py

@@ -3,10 +3,8 @@ the native server, then the files on disk compared with what the page made."""
 from pathlib import Path
 import os
 import shutil
-import socket
 import subprocess
 import tempfile
-import time
 import unittest
 
 import sys
@@ -38,11 +36,8 @@ class TransferBrowser(unittest.TestCase):
             shutil.copytree(ROOT / 'webui', root / 'webui', dirs_exist_ok=True)
             binary = Path(temp) / 'server'
             webui_build.build(binary, 'tests/webui_server_main.cpp', flags=('-O1',))
-            with socket.socket() as s:
+            server, port, _ = webui_build.start(binary, root)
-                s.bind(('127.0.0.1', 0)); port = s.getsockname()[1]
-            server = subprocess.Popen([str(binary), str(root), str(port)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
             try:
-                time.sleep(0.5)
                 try:
                     run = subprocess.run(['node', 'tests/webui_transfer_browser.cjs'], cwd=ROOT, capture_output=True, text=True, timeout=600,
                                          env={**os.environ, 'PLAYWRIGHT_PATH': PLAYWRIGHT, 'WEBUI_TEST_URL': f'http://127.0.0.1:{port}'})

tests/webui_build.py

@@ -1,7 +1,11 @@
 """Builds the WebUI's server for the host (the tests' fixtures): the given main, the
 server's sources and its libraries, with defines. One list of sources for every test."""
 from pathlib import Path
+import http.client
+import json
+import socket
 import subprocess
+import time
 
 ROOT = Path(__file__).resolve().parent.parent
 SOURCES = ['src/webui_ps5.cpp', 'src/webui_transfer.cpp', 'src/webui_update.cpp', 'src/scraper.cpp',
@@ -26,3 +30,31 @@ def build(output, main, defines=(), flags=('-O1', '-g')):
                     *[f'-D{d}' for d in defines], main, *SOURCES, *objects, http, update, '-lz', '-lcurl',
                     '-o', str(output)], cwd=ROOT, check=True)
     return output
+
+
+def start(binary, root, env=None, seconds=8.0):
+    """Starts a server built above (tests/webui_server_main.cpp) for the title folder root on a
+    free port, once it answers: its process, port and session token. A port found free can be
+    taken before the server binds it (another connection gets it as its own port): the server
+    then exits at once, "Address already in use", and starts again on another."""
+    for _ in range(5):
+        with socket.socket() as probe:
+            probe.bind(('127.0.0.1', 0))
+            port = probe.getsockname()[1]
+        process = subprocess.Popen([str(binary), str(root), str(port)], env=env,
+                                   stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        deadline = time.monotonic() + seconds
+        while process.poll() is None and time.monotonic() < deadline:
+            try:
+                connection = http.client.HTTPConnection('127.0.0.1', port, timeout=5)
+                connection.request('GET', '/api/status')
+                token = json.loads(connection.getresponse().read())['token']
+                connection.close()
+                return process, port, token
+            except (OSError, http.client.HTTPException):
+                time.sleep(0.02)
+        if process.poll() is None:  # running, and no answer: no port problem
+            process.kill()
+            process.wait()
+            break
+    raise RuntimeError('the server did not start')