Merge pull request #1435 from mrexodia/windows-build

Allow building on Windows without a Visual Studio command prompt

1 files changed, 106 insertions, 38 deletions

diff --git a/setup.py b/setup.py
index 52fc6f0a..11499b66 100644
--- a/setup.py
+++ b/setup.py
@@ -1,11 +1,14 @@
 #! /usr/bin/env python2
 
 from __future__ import print_function
-from distutils.core import setup, Extension
+# Reference: https://stackoverflow.com/a/13468644/1806760
+from setuptools import setup, Extension
 from distutils.util import get_platform
 from distutils.sysconfig import get_python_lib, get_config_vars
 from distutils.dist import DistributionMetadata
 from distutils.command.install_data import install_data
+from distutils.spawn import find_executable
+import subprocess
 from tempfile import TemporaryFile
 import fnmatch
 import io
@@ -15,6 +18,7 @@ from shutil import copy2, copyfile, rmtree
 import sys
 import tempfile
 import atexit
+import re
 
 is_win = platform.system() == "Windows"
 is_mac = platform.system() == "Darwin"
@@ -23,10 +27,10 @@ if is_win:
     import winreg
 
 def set_extension_compile_args(extension):
-    rel_lib_path = extension.name.replace('.', '/')
+    rel_lib_path = extension.name.replace(".", "/")
     abs_lib_path = os.path.join(get_python_lib(), rel_lib_path)
-    lib_name = abs_lib_path + '.so'
-    extension.extra_link_args = [ '-Wl,-install_name,' + lib_name]
+    lib_name = abs_lib_path + ".so"
+    extension.extra_link_args = [ "-Wl,-install_name," + lib_name]
 
 class smart_install_data(install_data):
     """Replacement for distutils.command.install_data to handle
@@ -47,41 +51,83 @@ def win_get_llvm_reg():
     except FileNotFoundError:
       pass
     return winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, REG_PATH, 0, winreg.KEY_READ)
-  
+
 def win_find_clang_path():
     try:
         with win_get_llvm_reg() as rkey:
             return winreg.QueryValueEx(rkey, None)[0]
     except FileNotFoundError:
+        # Visual Studio ships with an optional Clang distribution, try to detect it
+        clang_cl = find_executable("clang-cl")
+        if clang_cl is None:
+            return None
+        return os.path.abspath(os.path.join(os.path.dirname(clang_cl), "..", ".."))
+
+def win_get_clang_version(clang_path):
+    try:
+        clang_cl = os.path.join(clang_path, "bin", "clang.exe")
+        stdout = subprocess.check_output("\"{}\" --version".format(clang_cl))
+        version = stdout.splitlines(False)[0].decode()
+        match = re.search(r"version (\d+\.\d+\.\d+)", version)
+        if match is None:
+            return None
+        version = list(map(lambda s: int(s), match.group(1).split(".")))
+        return version
+    except FileNotFoundError:
         return None
 
 def win_use_clang():
-    # Recent (>= 8 ?) LLVM versions does not ship anymore a cl.exe binary in
-    # the msbuild-bin directory. Thus, we need to
-    # * copy-paste bin/clang-cl.exe into a temporary directory
-    # * rename it to cl.exe
-    # * add that path first in %Path%
-    # * clean this mess on exit
-    # We could use the build directory created by distutils for this, but it
-    # seems non trivial to gather
+    # To force python to use clang we copy the binaries in a temporary directory that's added to the PATH.
+    # We could use the build directory created by distutils for this, but it seems non-trivial to gather
     # (https://stackoverflow.com/questions/12896367/reliable-way-to-get-the-build-directory-from-within-setup-py).
+
     clang_path = win_find_clang_path()
     if clang_path is None:
         return False
+    clang_version = win_get_clang_version(clang_path)
+    if clang_version is None:
+        return False
     tmpdir = tempfile.mkdtemp(prefix="llvm")
+
     copyfile(os.path.join(clang_path, "bin", "clang-cl.exe"), os.path.join(tmpdir, "cl.exe"))
-    os.environ['Path'] = "%s;%s" % (tmpdir, os.environ["Path"])
-    atexit.register(lambda dir_: rmtree(dir_), tmpdir)
 
+    # If you run the installation from a Visual Studio command prompt link.exe will already exist
+    # Fall back to LLVM's lld-link.exe which is compatible with link's command line
+    if find_executable("link") is None:
+        # LLVM >= 14.0.0 started supporting the /LTCG flag
+        # Earlier versions will error during the linking phase so bail out now
+        if clang_version[0] < 14:
+            return False
+        copyfile(os.path.join(clang_path, "bin", "lld-link.exe"), os.path.join(tmpdir, "link.exe"))
+
+    # Add the temporary directory at the front of the PATH and clean up on exit
+    os.environ["PATH"] = "%s;%s" % (tmpdir, os.environ["PATH"])
+    atexit.register(lambda dir_: rmtree(dir_), tmpdir)
+    print("Found Clang {}.{}.{}: {}".format(clang_version[0], clang_version[1], clang_version[2], clang_path))
     return True
 
+build_extensions = True
+build_warnings = []
 win_force_clang = False
-if is_win and is_64bit:
-    # We do not change to clang if under 32 bits, because even with Clang we
-    # don't have uint128_t with the 32 bits ABI.
-    win_force_clang = win_use_clang()
-    if not win_force_clang:
-        print("Warning: couldn't find a Clang/LLVM installation. Some runtime functions needed by the jitter won't be compiled.")
+if is_win:
+    if is_64bit or find_executable("cl") is None:
+        # We do not change to clang if under 32 bits, because even with Clang we
+        # do not use uint128_t with the 32 bits ABI. Regardless we can try to
+        # find it when building in 32-bit mode if cl.exe was not found in the PATH.
+        win_force_clang = win_use_clang()
+        if is_64bit and not win_force_clang:
+            build_warnings.append("Could not find a suitable Clang/LLVM installation. You can download LLVM from https://releases.llvm.org")
+            build_warnings.append("Alternatively you can select the 'C++ Clang-cl build tools' in the Visual Studio Installer")
+            build_extensions = False
+    cl = find_executable("cl")
+    link = find_executable("link")
+    if cl is None or link is None:
+        build_warnings.append("Could not find cl.exe and/or link.exe in the PATH, try building miasm from a Visual Studio command prompt")
+        build_warnings.append("More information at: https://wiki.python.org/moin/WindowsCompilers")
+        build_extensions = False
+    else:
+        print("Found cl.exe: {}".format(cl))
+        print("Found link.exe: {}".format(link))
 
 def build_all():
     packages=[
@@ -223,29 +269,42 @@ def build_all():
 
     if is_win:
         # Force setuptools to use whatever msvc version installed
-        os.environ['MSSdk'] = '1'
-        os.environ['DISTUTILS_USE_SDK'] = '1'
+        # https://docs.python.org/3/distutils/apiref.html#module-distutils.msvccompiler
+        os.environ["MSSdk"] = "1"
+        os.environ["DISTUTILS_USE_SDK"] = "1"
+        extra_compile_args = ["-D_CRT_SECURE_NO_WARNINGS"]
         if win_force_clang:
             march = "-m64" if is_64bit else "-m32"
-            for extension in ext_modules_all:
-                extension.extra_compile_args = [march]
+            extra_compile_args += [
+                march,
+                "-Wno-unused-command-line-argument",
+                "-Wno-visibility",
+                "-Wno-dll-attribute-on-redeclaration",
+                "-Wno-tautological-compare",
+                "-Wno-unused-but-set-variable",
+            ]
+        for extension in ext_modules_all:
+            extension.extra_compile_args = extra_compile_args
     elif is_mac:
         for extension in ext_modules_all:
             set_extension_compile_args(extension)
         cfg_vars = get_config_vars()
-        cfg_vars['LDSHARED'] = cfg_vars['LDSHARED'].replace('-bundle', '-dynamiclib')
+        cfg_vars["LDSHARED"] = cfg_vars["LDSHARED"].replace("-bundle", "-dynamiclib")
+
+    # Do not attempt to build the extensions when disabled
+    if not build_extensions:
+        ext_modules_all = []
 
     print("building")
     build_ok = False
-    for name, ext_modules in [("all", ext_modules_all),
-    ]:
+    for name, ext_modules in [("all", ext_modules_all)]:
         print("build with", repr(name))
         try:
             s = setup(
                 name = "miasm",
                 version = __import__("miasm").VERSION,
                 packages = packages,
-                data_files=[('', ["README.md"])],
+                data_files=[("", ["README.md"])],
                 package_data = {
                     "miasm": [
                         "jitter/*.h",
@@ -253,7 +312,7 @@ def build_all():
                         "VERSION"
                     ]
                 },
-                install_requires=['future', 'pyparsing~=2.0'],
+                install_requires=["future", "pyparsing~=2.0"],
                 cmdclass={"install_data": smart_install_data},
                 ext_modules = ext_modules,
                 # Metadata
@@ -285,6 +344,10 @@ def build_all():
         build_ok = True
         break
     if not build_ok:
+        if len(build_warnings) > 0:
+            print("ERROR: There was an issue setting up the build environment:")
+            for warning in build_warnings:
+                print("  " + warning)
         raise ValueError("Unable to build Miasm!")
     print("build", name)
     # we copy libraries from build dir to current miasm directory
@@ -294,7 +357,7 @@ def build_all():
             build_base = s.command_options["build"]["build_base"]
 
     print(build_base)
-    if is_win:
+    if is_win and build_extensions:
         libs = []
         for root, _, files in os.walk(build_base):
             for filename in files:
@@ -322,11 +385,16 @@ def build_all():
                 print("Copying", lib, "to", dst)
                 copy2(lib, dst)
 
+    # Inform the user about the skipped build
+    if not build_extensions:
+        print("WARNING: miasm jit extensions were not compiled, details:")
+        for warning in build_warnings:
+            print("  " + warning)
 
-with io.open(os.path.join(os.path.abspath(os.path.dirname('__file__')),
-                       'README.md'), encoding='utf-8') as fdesc:
+with io.open(os.path.join(os.path.abspath(os.path.dirname("__file__")),
+                       "README.md"), encoding="utf-8") as fdesc:
     long_description = fdesc.read()
-long_description_content_type = 'text/markdown'
+long_description_content_type = "text/markdown"
 
 
 # Monkey patching (distutils does not handle Description-Content-Type
@@ -339,10 +407,10 @@ def _write_pkg_file(self, file):
         _write_pkg_file_orig(self, tmpfd)
         tmpfd.seek(0)
         for line in tmpfd:
-            if line.startswith('Metadata-Version: '):
-                file.write('Metadata-Version: 2.1\n')
-            elif line.startswith('Description: '):
-                file.write('Description-Content-Type: %s; charset=UTF-8\n' %
+            if line.startswith("Metadata-Version: "):
+                file.write("Metadata-Version: 2.1\n")
+            elif line.startswith("Description: "):
+                file.write("Description-Content-Type: %s; charset=UTF-8\n" %
                            long_description_content_type)
                 file.write(line)
             else: