setup_pydevd_cython.py

"""
A simpler setup version just to compile the speedup module.

It should be used as:

python setup_pydevd_cython build_ext --inplace

Note: the .c file and other generated files are regenerated from
the .pyx file by running "python build_tools/build.py"
"""

import os
import sys
from setuptools import setup

os.chdir(os.path.dirname(os.path.abspath(__file__)))

IS_PY36_OR_GREATER = sys.version_info[:2] >= (3, 6)
IS_PY311_ONWARDS = sys.version_info[:2] >= (3, 11)
IS_PY312_ONWARDS = sys.version_info[:2] >= (3, 12)


def process_args():
    extension_folder = None
    target_pydevd_name = None
    target_frame_eval = None
    force_cython = False

    for i, arg in enumerate(sys.argv[:]):
        if arg == "--build-lib":
            extension_folder = sys.argv[i + 1]
            # It shouldn't be removed from sys.argv (among with --build-temp) because they're passed further to setup()
        if arg.startswith("--target-pyd-name="):
            sys.argv.remove(arg)
            target_pydevd_name = arg[len("--target-pyd-name=") :]
        if arg.startswith("--target-pyd-frame-eval="):
            sys.argv.remove(arg)
            target_frame_eval = arg[len("--target-pyd-frame-eval=") :]
        if arg == "--force-cython":
            sys.argv.remove(arg)
            force_cython = True

    return extension_folder, target_pydevd_name, target_frame_eval, force_cython


def process_template_lines(template_lines):
    # Create 2 versions of the template, one for Python 3.8 and another for Python 3.9
    for version in ("38", "39"):
        yield "### WARNING: GENERATED CODE, DO NOT EDIT!"
        yield "### WARNING: GENERATED CODE, DO NOT EDIT!"
        yield "### WARNING: GENERATED CODE, DO NOT EDIT!"

        for line in template_lines:
            if version == "38":
                line = line.replace(
                    "get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc)",
                    "get_bytecode_while_frame_eval_38(PyFrameObject * frame_obj, int exc)",
                )
                line = line.replace("CALL_EvalFrameDefault", "CALL_EvalFrameDefault_38(frame_obj, exc)")
            else:  # 3.9
                line = line.replace(
                    "get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc)",
                    "get_bytecode_while_frame_eval_39(PyThreadState* tstate, PyFrameObject * frame_obj, int exc)",
                )
                line = line.replace("CALL_EvalFrameDefault", "CALL_EvalFrameDefault_39(tstate, frame_obj, exc)")

            yield line

        yield "### WARNING: GENERATED CODE, DO NOT EDIT!"
        yield "### WARNING: GENERATED CODE, DO NOT EDIT!"
        yield "### WARNING: GENERATED CODE, DO NOT EDIT!"
        yield ""
        yield ""


def process_template_file(contents):
    ret = []
    template_lines = []

    append_to = ret
    for line in contents.splitlines(keepends=False):
        if line.strip() == "### TEMPLATE_START":
            append_to = template_lines
        elif line.strip() == "### TEMPLATE_END":
            append_to = ret
            for line in process_template_lines(template_lines):
                ret.append(line)
        else:
            append_to.append(line)

    return "\n".join(ret)


def build_extension(dir_name, extension_name, target_pydevd_name, force_cython, extended=False, has_pxd=False, template=False):
    pyx_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.pyx" % (extension_name,))

    if template:
        pyx_template_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.template.pyx" % (extension_name,))
        with open(pyx_template_file, "r") as stream:
            contents = stream.read()

        contents = process_template_file(contents)

        with open(pyx_file, "w") as stream:
            stream.write(contents)

    if target_pydevd_name != extension_name:
        # It MUST be there in this case!
        # (otherwise we'll have unresolved externals because the .c file had another name initially).
        import shutil

        # We must force cython in this case (but only in this case -- for the regular setup in the user machine, we
        # should always compile the .c file).
        force_cython = True

        new_pyx_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.pyx" % (target_pydevd_name,))
        new_c_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.c" % (target_pydevd_name,))
        shutil.copy(pyx_file, new_pyx_file)
        pyx_file = new_pyx_file
        if has_pxd:
            pxd_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.pxd" % (extension_name,))
            new_pxd_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.pxd" % (target_pydevd_name,))
            shutil.copy(pxd_file, new_pxd_file)
        assert os.path.exists(pyx_file)

    try:
        c_files = [
            os.path.join(dir_name, "%s.c" % target_pydevd_name),
        ]
        if force_cython:
            for c_file in c_files:
                try:
                    os.remove(c_file)
                except:
                    pass
            from Cython.Build import cythonize  # @UnusedImport
            # Generate the .c files in cythonize (will not compile at this point).

            target = "%s/%s.pyx" % (
                dir_name,
                target_pydevd_name,
            )
            cythonize([target])

            # Workarounds needed in CPython 3.8 and 3.9 to access PyInterpreterState.eval_frame.
            for c_file in c_files:
                with open(c_file, "r") as stream:
                    c_file_contents = stream.read()

                if '#include "internal/pycore_gc.h"' not in c_file_contents:
                    c_file_contents = c_file_contents.replace(
                        '#include "Python.h"',
                        """#include "Python.h"
#if PY_VERSION_HEX >= 0x03090000
#include "internal/pycore_gc.h"
#include "internal/pycore_interp.h"
#endif
""",
                    )

                if '#include "internal/pycore_pystate.h"' not in c_file_contents:
                    c_file_contents = c_file_contents.replace(
                        '#include "pystate.h"',
                        """#include "pystate.h"
#if PY_VERSION_HEX >= 0x03080000
#include "internal/pycore_pystate.h"
#endif
""",
                    )

                # We want the same output on Windows and Linux.
                c_file_contents = c_file_contents.replace("\r\n", "\n").replace("\r", "\n")
                c_file_contents = c_file_contents.replace(r"_pydevd_frame_eval\\release_mem.h", "_pydevd_frame_eval/release_mem.h")
                c_file_contents = c_file_contents.replace(
                    r"_pydevd_frame_eval\\pydevd_frame_evaluator.pyx", "_pydevd_frame_eval/pydevd_frame_evaluator.pyx"
                )
                c_file_contents = c_file_contents.replace(r"_pydevd_bundle\\pydevd_cython.pxd", "_pydevd_bundle/pydevd_cython.pxd")
                c_file_contents = c_file_contents.replace(r"_pydevd_bundle\\pydevd_cython.pyx", "_pydevd_bundle/pydevd_cython.pyx")

                with open(c_file, "w") as stream:
                    stream.write(c_file_contents)

        # Always compile the .c (and not the .pyx) file (which we should keep up-to-date by running build_tools/build.py).
        from distutils.extension import Extension

        extra_compile_args = []
        extra_link_args = []

        if "linux" in sys.platform:
            # Enabling -flto brings executable from 4MB to 0.56MB and -Os to 0.41MB
            # Profiling shows an execution around 3-5% slower with -Os vs -O3,
            # so, kept only -flto.
            extra_compile_args = ["-flto", "-O3"]
            extra_link_args = extra_compile_args[:]

            # Note: also experimented with profile-guided optimization. The executable
            # size became a bit smaller (from 0.56MB to 0.5MB) but this would add an
            # extra step to run the debugger to obtain the optimizations
            # so, skipped it for now (note: the actual benchmarks time was in the
            # margin of a 0-1% improvement, which is probably not worth it for
            # speed increments).
            # extra_compile_args = ["-flto", "-fprofile-generate"]
            # ... Run benchmarks ...
            # extra_compile_args = ["-flto", "-fprofile-use", "-fprofile-correction"]
        elif "win32" in sys.platform:
            pass
            # uncomment to generate pdbs for visual studio.
            # extra_compile_args=["-Zi", "/Od"]
            # extra_link_args=["-debug"]
            if IS_PY311_ONWARDS:
                # On py311 we need to add the CPython include folder to the include path.
                extra_compile_args.append("-I%s\\include\\CPython" % sys.exec_prefix)

        kwargs = {}
        if extra_link_args:
            kwargs["extra_link_args"] = extra_link_args
        if extra_compile_args:
            kwargs["extra_compile_args"] = extra_compile_args

        ext_modules = [
            Extension(
                "%s%s.%s"
                % (
                    dir_name,
                    "_ext" if extended else "",
                    target_pydevd_name,
                ),
                c_files,
                **kwargs,
            )
        ]

        # This is needed in CPython 3.8 to be able to include internal/pycore_pystate.h
        # (needed to set PyInterpreterState.eval_frame).
        for module in ext_modules:
            module.define_macros = [("Py_BUILD_CORE_MODULE", "1")]
        setup(name="Cythonize", ext_modules=ext_modules)
    finally:
        if target_pydevd_name != extension_name:
            try:
                os.remove(new_pyx_file)
            except:
                import traceback

                traceback.print_exc()
            try:
                os.remove(new_c_file)
            except:
                import traceback

                traceback.print_exc()
            if has_pxd:
                try:
                    os.remove(new_pxd_file)
                except:
                    import traceback

                    traceback.print_exc()


extension_folder, target_pydevd_name, target_frame_eval, force_cython = process_args()

FORCE_BUILD_ALL = os.environ.get("PYDEVD_FORCE_BUILD_ALL", "").lower() in ("true", "1")

extension_name = "pydevd_cython"
if target_pydevd_name is None:
    target_pydevd_name = extension_name
build_extension("_pydevd_bundle", extension_name, target_pydevd_name, force_cython, extension_folder, True)

if IS_PY36_OR_GREATER and not IS_PY311_ONWARDS or FORCE_BUILD_ALL:
    extension_name = "pydevd_frame_evaluator"
    if target_frame_eval is None:
        target_frame_eval = extension_name
    build_extension("_pydevd_frame_eval", extension_name, target_frame_eval, force_cython, extension_folder, True, template=True)

if IS_PY312_ONWARDS or FORCE_BUILD_ALL:
    extension_name = "_pydevd_sys_monitoring_cython"
    build_extension("_pydevd_sys_monitoring", extension_name, extension_name, force_cython, extension_folder, True)

if extension_folder:
    os.chdir(extension_folder)
    for folder in [
        file for file in os.listdir(extension_folder) if file != "build" and os.path.isdir(os.path.join(extension_folder, file))
    ]:
        file = os.path.join(folder, "__init__.py")
        if not os.path.exists(file):
            open(file, "a").close()