From 160df614083997dc8f03750ff3ceaff458fa3fe0 Mon Sep 17 00:00:00 2001 From: Martin Quinson Date: Wed, 5 Jun 2019 11:57:27 +0200 Subject: [PATCH] Build nice python source packages This should be enough to solve framagit#6. --- ChangeLog | 4 + MANIFEST.in.in | 3 + doc/doxygen/inside_release.doc | 9 ++- docs/source/Installing_SimGrid.rst | 35 ++++++--- setup.py | 113 +++++++++++++++++++++++++++++ tools/cmake/DefinePackages.cmake | 3 + tools/cmake/Distrib.cmake | 8 +- 7 files changed, 164 insertions(+), 11 deletions(-) create mode 100644 MANIFEST.in.in create mode 100755 setup.py diff --git a/ChangeLog b/ChangeLog index 769b6745a1..55033f4bb8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25,10 +25,14 @@ Network models: - Remove the lagrange-based models (Reno/Reno2/Vegas). The regular models proved to be more accurate than these old experiments. +Python: + - Simgrid can now hopefully be installed with pip. + Fixed bugs (FG=FramaGit; GH=GitHub): - FG#1: Broken link in error messages - FG#2: missing installation documentation - FG#3: missing documentation in smpirun + - FG#6: Python bindings not available on PyPI - FG#7: simple cmake call requires doxygen - FG#8: make python bindings an optional dependency - FG#10: Can not use MSG_process_set_data from SMPI any more diff --git a/MANIFEST.in.in b/MANIFEST.in.in new file mode 100644 index 0000000000..9c36a1405f --- /dev/null +++ b/MANIFEST.in.in @@ -0,0 +1,3 @@ +# This file lists the content of the python source package +# Prepared in tools/cmake/Distrib.cmake +@PYTHON_SOURCES@ diff --git a/doc/doxygen/inside_release.doc b/doc/doxygen/inside_release.doc index 0442bc6ac0..d6da2b8fec 100644 --- a/doc/doxygen/inside_release.doc +++ b/doc/doxygen/inside_release.doc @@ -171,14 +171,21 @@ Doing a dot release: - CMakeLists.txt (in macros SIMGRID_VERSION_*) - sonar-project.properties - docs/source/conf.py + - setup.py - Commit and push to both framagit and github - Wait for both appveyor and jenkins/highsierra to complete the build - If it's not successful, fix it and push again - - Once it's successful: + - Once it's successful on both appveyor and jenkins: - tag the release v3_X_Y and push the tag - Merge 'master' into 'stable' and push it to framagit - Do not merge into 'stable' before appveyor and jenkins are done, or your jarfile will not contain the code you expect for win and mac. + - python3 setup.py sdist # Build a source distrib + - test that the built distrib recompiles: + rm -rf /tmp/pysimgrid && mkdir /tmp/pysimgrid && cp dist/simgrid-*.tar.gz /tmp/pysimgrid + (cd /tmp/pysimgrid && tar xfz simgrid*.tgz && cd simgrid-* && python3 setup.py build) + - Upload it to pypi (WARNING: you cannot modify uploaded files, ever) + twine upload dist/simgrid-*.tar.gz */ diff --git a/docs/source/Installing_SimGrid.rst b/docs/source/Installing_SimGrid.rst index 8a5218a6b5..31cb7e9b70 100644 --- a/docs/source/Installing_SimGrid.rst +++ b/docs/source/Installing_SimGrid.rst @@ -348,8 +348,8 @@ on `our Jenkins `_. .. _install_cmake_mac: -macOS Specifics -^^^^^^^^^^^^^^^ +macOS-specific instructions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ SimGrid compiles like a charm with clang (version 3.0 or higher) on macOS: @@ -376,8 +376,8 @@ CMake Error: Parse error in cache file build_dir/CMakeCache.txt. Offending entry .. _install_cmake_windows: -Windows Specifics -^^^^^^^^^^^^^^^^^ +Windows-specific instructions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The best solution to get SimGrid working on windows is to install the Ubuntu subsystem of Windows 10. All of SimGrid (but the model-checker) @@ -394,15 +394,32 @@ with MS Visual C. Some clang-based tools seem promising to fix this, but this is of rather low priority for us. It it's important for you and if you get it working, please @ref community_contact "tell us". -Java Specifics -^^^^^^^^^^^^^^ +Python-specific instructions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Recompiling the python bindings from the source should be as easy as: + +.. code-block:: shell + + # cd simgrid-source-tree + python setup.py build install + +Starting with SimGrid 3.13, it should even be possible to install +simgrid without downloading the source with pip: + +.. code-block:: shell + + pip install simgrid + +Java-specific instructions +^^^^^^^^^^^^^^^^^^^^^^^^^^ Once you have the `full JDK `_ installed, things should be as simple as: .. code-block:: shell - cmake -Denable_java=ON . + cmake -Denable_java=ON -Dminimal-bindings=ON . make simgrid-java_jar # Only build the jarfile After the compilation, the file ```simgrid.jar``` is produced in the @@ -433,8 +450,8 @@ path, and relaunch cmake. If you have several versions of JNI installed Note that the filename ```jni.h``` was removed from the path. -Linux Multi-Arch Specifics -^^^^^^^^^^^^^^^^^^^^^^^^^^ +Linux Multi-Arch specific instructions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ On a multiarch x86_64 Linux, it should be possible to compile a 32-bit version of SimGrid with something like: diff --git a/setup.py b/setup.py new file mode 100755 index 0000000000..f3b07d5563 --- /dev/null +++ b/setup.py @@ -0,0 +1,113 @@ + + +# python3 setup.py sdist # Build a source distrib (building binary distribs is complex on linux) + +# twine upload --repository-url https://test.pypi.org/legacy/ dist/simgrid-*.tar.gz # Upload to test +# pip3 install --user --index-url https://test.pypi.org/simple simgrid + +# Once it works, upload to the real infra. /!\ you cannot modify a file once uploaded +# twine upload dist/simgrid-*.tar.gz + +import os +import re +import sys +import platform +import subprocess + +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext +from distutils.version import LooseVersion + + +class CMakeExtension(Extension): + def __init__(self, name, sourcedir=''): + Extension.__init__(self, name, sources=[]) + self.sourcedir = os.path.abspath(sourcedir) + + +class CMakeBuild(build_ext): + def run(self): + try: + out = subprocess.check_output(['cmake', '--version']) + except OSError: + raise RuntimeError("CMake must be installed to build the following extensions: " + + ", ".join(e.name for e in self.extensions)) + if not os.path.exists("MANIFEST.in"): + raise RuntimeError("Please generate a MANIFEST.in file (configure simgrid, and copy it here if you build out of tree)") + + if platform.system() == "Windows": + cmake_version = LooseVersion(re.search(r'version\s*([\d.]+)', out.decode()).group(1)) + if cmake_version < '3.1.0': + raise RuntimeError("CMake >= 3.1.0 is required on Windows") + + for ext in self.extensions: + self.build_extension(ext) + + def build_extension(self, ext): + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir, + '-DPYTHON_EXECUTABLE=' + sys.executable, + '-Denable_smpi=OFF', + '-Denable_java=OFF', + '-Denable_python=ON', + '-Dminimal-bindings=ON'] + + cfg = 'Debug' if self.debug else 'Release' + build_args = ['--config', cfg] + + if platform.system() == "Windows": + cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), extdir)] + if sys.maxsize > 2**32: + cmake_args += ['-A', 'x64'] + build_args += ['--', '/m'] + else: + cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] + build_args += ['--', '-j4'] + + env = os.environ.copy() + env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''), + self.distribution.get_version()) + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env) + subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp) + +setup( + name='simgrid', + version='3.22.2', + author='Da SimGrid Team', + author_email='simgrid-devel@lists.gforge.inria.fr', + description='Toolkit for scalable simulation of distributed applications', + long_description=("SimGrid is a scientific instrument to study the behavior of " + "large-scale distributed systems such as Grids, Clouds, HPC or P2P " + "systems. It can be used to evaluate heuristics, prototype applications " + "or even assess legacy MPI applications."), + ext_modules=[CMakeExtension('simgrid')], + cmdclass=dict(build_ext=CMakeBuild), + zip_safe=False, + classifiers=[ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Education", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)", + "Operating System :: POSIX", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Programming Language :: Python :: 3", + "Programming Language :: C++", + "Programming Language :: C", + "Programming Language :: Fortran", + "Programming Language :: Java", + "Topic :: System :: Distributed Computing", + "Topic :: System :: Systems Administration", + ], + url="https://simgrid.org", + project_urls={ + 'Tracker': 'https://framagit.org/simgrid/simgrid/issues/', + 'Source': 'https://framagit.org/simgrid/simgrid/', + 'Documentation': 'https://simgrid.org/doc/latest/', + }, +) diff --git a/tools/cmake/DefinePackages.cmake b/tools/cmake/DefinePackages.cmake index d8fe6cca94..a70f4fde0d 100644 --- a/tools/cmake/DefinePackages.cmake +++ b/tools/cmake/DefinePackages.cmake @@ -1081,6 +1081,9 @@ set(CMAKE_SOURCE_FILES tools/stack-cleaner/clean-stack-filter tools/stack-cleaner/compiler-wrapper tools/stack-cleaner/README + + setup.py + MANIFEST.in.in ) set(PLATFORMS_EXAMPLES diff --git a/tools/cmake/Distrib.cmake b/tools/cmake/Distrib.cmake index 652b572813..b6e5531c0a 100644 --- a/tools/cmake/Distrib.cmake +++ b/tools/cmake/Distrib.cmake @@ -177,7 +177,6 @@ set(source_to_pack ${examples_src} ${tesh_files} ${teshsuite_src} - ${testsuite_src} ${tools_src} ${txt_files} ${xml_files} @@ -206,10 +205,14 @@ if(IS_DIRECTORY ${CMAKE_BINARY_DIR}/doc/html/) endif() set(dirs_in_tarball "") +set(PYTHON_SOURCES "include MANIFEST.in") foreach(file ${source_to_pack}) #message(${file}) # This damn prefix is still set somewhere (seems to be in subdirs) string(REPLACE "${CMAKE_HOME_DIRECTORY}/" "" file "${file}") + + # Prepare the list of files to include in the python sdist, one per line + set(PYTHON_SOURCES "${PYTHON_SOURCES}\ninclude ${file}") # Create the directory on need get_filename_component(file_location ${file} PATH) @@ -226,6 +229,9 @@ foreach(file ${source_to_pack}) TARGET dist-dir COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_HOME_DIRECTORY}/${file} ${PROJECT_NAME}-${release_version}/${file_location}) endforeach(file ${source_to_pack}) +configure_file("${CMAKE_HOME_DIRECTORY}/MANIFEST.in.in" "${CMAKE_BINARY_DIR}/MANIFEST.in" @ONLY IMMEDIATE) +unset(PYTHON_SOURCES) + add_custom_command( TARGET dist-dir -- 2.20.1