From 49db24ce273ebccb598fc13de946d1ac3cad38f8 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 10:30:29 +0300 Subject: [PATCH 1/6] launch.py: Add debugging envvar to see install output --- launch.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/launch.py b/launch.py index cfc0cffa2..136e8855c 100644 --- a/launch.py +++ b/launch.py @@ -22,6 +22,9 @@ stored_commit_hash = None stored_git_tag = None dir_repos = "repositories" +# Whether to default to printing command output +default_command_live = (os.environ.get('WEBUI_LAUNCH_LIVE_OUTPUT') == "1") + if 'GRADIO_ANALYTICS_ENABLED' not in os.environ: os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False' @@ -85,7 +88,7 @@ def git_tag(): return stored_git_tag -def run(command, desc=None, errdesc=None, custom_env=None, live=False): +def run(command, desc=None, errdesc=None, custom_env=None, live: bool = default_command_live): if desc is not None: print(desc) @@ -135,7 +138,7 @@ def run_python(code, desc=None, errdesc=None): return run(f'"{python}" -c "{code}"', desc, errdesc) -def run_pip(command, desc=None, live=False): +def run_pip(command, desc=None, live=default_command_live): if args.skip_install: return From 875bc270093b4f89b9d0d20a93a5e8ea514b3f6c Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 10:34:19 +0300 Subject: [PATCH 2/6] launch.py: Simplify run() --- launch.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/launch.py b/launch.py index 136e8855c..ae75f6fda 100644 --- a/launch.py +++ b/launch.py @@ -88,32 +88,36 @@ def git_tag(): return stored_git_tag -def run(command, desc=None, errdesc=None, custom_env=None, live: bool = default_command_live): +def run(command, desc=None, errdesc=None, custom_env=None, live: bool = default_command_live) -> str: if desc is not None: print(desc) - if live: - result = subprocess.run(command, shell=True, env=os.environ if custom_env is None else custom_env) - if result.returncode != 0: - raise RuntimeError(f"""{errdesc or 'Error running command'}. -Command: {command} -Error code: {result.returncode}""") + run_kwargs = { + "args": command, + "shell": True, + "env": os.environ if custom_env is None else custom_env, + "encoding": 'utf8', + "errors": 'ignore', + } - return "" + if not live: + run_kwargs["stdout"] = run_kwargs["stderr"] = subprocess.PIPE - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ if custom_env is None else custom_env) + result = subprocess.run(**run_kwargs) if result.returncode != 0: + error_bits = [ + f"{errdesc or 'Error running command'}.", + f"Command: {command}", + f"Error code: {result.returncode}", + ] + if result.stdout: + error_bits.append(f"stdout: {result.stdout}") + if result.stderr: + error_bits.append(f"stderr: {result.stderr}") + raise RuntimeError("\n".join(error_bits)) - message = f"""{errdesc or 'Error running command'}. -Command: {command} -Error code: {result.returncode} -stdout: {result.stdout.decode(encoding="utf8", errors="ignore") if len(result.stdout)>0 else ''} -stderr: {result.stderr.decode(encoding="utf8", errors="ignore") if len(result.stderr)>0 else ''} -""" - raise RuntimeError(message) - - return result.stdout.decode(encoding="utf8", errors="ignore") + return (result.stdout or "") def check_run(command): From a09e1e6e18e800191817493253dfc481a07b1868 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 10:48:48 +0300 Subject: [PATCH 3/6] launch.py: Use GitHub archive URLs for gfpgan, clip, openclip instead of git clones --- launch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/launch.py b/launch.py index ae75f6fda..fed14d93a 100644 --- a/launch.py +++ b/launch.py @@ -248,9 +248,9 @@ def prepare_environment(): requirements_file = os.environ.get('REQS_FILE', "requirements_versions.txt") xformers_package = os.environ.get('XFORMERS_PACKAGE', 'xformers==0.0.17') - gfpgan_package = os.environ.get('GFPGAN_PACKAGE', "git+https://github.com/TencentARC/GFPGAN.git@8d2447a2d918f8eba5a4a01463fd48e45126a379") - clip_package = os.environ.get('CLIP_PACKAGE', "git+https://github.com/openai/CLIP.git@d50d76daa670286dd6cacf3bcd80b5e4823fc8e1") - openclip_package = os.environ.get('OPENCLIP_PACKAGE', "git+https://github.com/mlfoundations/open_clip.git@bb6e834e9c70d9c27d0dc3ecedeebeaeb1ffad6b") + gfpgan_package = os.environ.get('GFPGAN_PACKAGE', "https://github.com/TencentARC/GFPGAN/archive/8d2447a2d918f8eba5a4a01463fd48e45126a379.zip") + clip_package = os.environ.get('CLIP_PACKAGE', "https://github.com/openai/CLIP/archive/d50d76daa670286dd6cacf3bcd80b5e4823fc8e1.zip") + openclip_package = os.environ.get('OPENCLIP_PACKAGE', "https://github.com/mlfoundations/open_clip/archive/bb6e834e9c70d9c27d0dc3ecedeebeaeb1ffad6b.zip") stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git") From dd3ca9adf7077cb7209b31fd16a40deffc4948ca Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 10:35:22 +0300 Subject: [PATCH 4/6] launch.py: make torch_index_url an envvar --- launch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/launch.py b/launch.py index fed14d93a..670af87c1 100644 --- a/launch.py +++ b/launch.py @@ -244,7 +244,8 @@ def run_extensions_installers(settings_file): def prepare_environment(): - torch_command = os.environ.get('TORCH_COMMAND', "pip install torch==2.0.1 torchvision==0.15.2 --extra-index-url https://download.pytorch.org/whl/cu118") + torch_index_url = os.environ.get('TORCH_INDEX_URL', "https://download.pytorch.org/whl/cu118") + torch_command = os.environ.get('TORCH_COMMAND', f"pip install torch==2.0.1 torchvision==0.15.2 --extra-index-url {torch_index_url}") requirements_file = os.environ.get('REQS_FILE', "requirements_versions.txt") xformers_package = os.environ.get('XFORMERS_PACKAGE', 'xformers==0.0.17') From c702010e575d68fff0b3e640e10b6e750513b5a8 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 10:36:10 +0300 Subject: [PATCH 5/6] CI: use CPU wheel repo for PyTorch --- .github/workflows/run_tests.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 9a0b8d22b..58bc4770c 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -19,6 +19,11 @@ jobs: **/requirements*txt - name: Run tests run: python launch.py --tests test --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test + env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PIP_PROGRESS_BAR: "off" + TORCH_INDEX_URL: https://download.pytorch.org/whl/cpu + WEBUI_LAUNCH_LIVE_OUTPUT: "1" - name: Upload main app stdout-stderr uses: actions/upload-artifact@v3 if: always() From 5b592669f9917cf885b0f597ac427ba91c847801 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 10:51:52 +0300 Subject: [PATCH 6/6] CI: use launch.py for dependencies too --- .github/workflows/run_tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 58bc4770c..0708398b9 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -17,6 +17,7 @@ jobs: cache: pip cache-dependency-path: | **/requirements*txt + launch.py - name: Run tests run: python launch.py --tests test --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test env: