diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b95fc2a5a..d939093ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,7 +75,7 @@ jobs: libeigen3-dev - name: Install python dependencies - run: pip install -r dev_requirements.txt + run: pip install -r dev_requirements_$(${{ matrix.python }} -c 'import sys; print(f"{sys.version_info.major}{sys.version_info.minor}")').txt - name: Run cmake build run: | @@ -87,14 +87,12 @@ jobs: cmake --build build -j $(nproc) # - lcmtypes need to be available for tests - # - Exact contents of dev_requirements.txt depend on python version. Need to update file to - # match current python version to avoid failure of corresponding gen test. symforce needs - # to be on the PYTHONPATH to run gen test in this manner. + # TODO(brad): Check that it is still necessary for $(pwd) to be on the PYTHONPATH + # - symforce needs to be on the PYTHONPATH to run gen test in this manner. - name: Run tests run: | pip install build/lcmtypes/python2.7 export PYTHONPATH=$PYTHONPATH:$(pwd) - ${{ matrix.python }} test/symforce_requirements_test.py --update ctest --test-dir build -j $(nproc) || true ctest --test-dir build -j $(nproc) --rerun-failed --output-on-failure diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 754c86461..1022ea5d9 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,7 +23,7 @@ jobs: pandoc - name: Install python dependencies - run: pip install -r dev_requirements.txt + run: pip install -r dev_requirements_$(python3 -c 'import sys; print(f"{sys.version_info.major}{sys.version_info.minor}")').txt - name: Run cmake build run: | diff --git a/README.md b/README.md index 1873d1ea5..20075db7c 100644 --- a/README.md +++ b/README.md @@ -565,7 +565,7 @@ pip install -e . This will build the C++ components of SymForce, but you won't be able to run `pip install -e .` repeatedly if you need to rebuild C++ code. If you're changing C++ code and rebuilding, you should build with CMake directly as described below. -`pip install .` will not install pinned versions of SymForce's dependencies, it'll install any compatible versions. It also won't install all packages required to run all of the SymForce tests and build all of the targets (e.g. building the docs or running the linters). If you want all packages required for that, you should `pip install .[dev]` instead (or one of the other groups of extra requirements in our `setup.py`). If you additionally want pinned versions of our dependencies, which are the exact versions guaranteed by CI to pass all of our tests, you can install them from `pip install -r dev_requirements.txt`. +`pip install .` will not install pinned versions of SymForce's dependencies, it'll install any compatible versions. It also won't install all packages required to run all of the SymForce tests and build all of the targets (e.g. building the docs or running the linters). If you want all packages required for that, you should `pip install .[dev]` instead (or one of the other groups of extra requirements in our `setup.py`). If you additionally want pinned versions of our dependencies, which are the exact versions guaranteed by CI to pass all of our tests, you can install them from `pip install -r dev_requirements_XY.txt` (where `X` and `Y` are the major and minor versions of your python; e.g. `dev_requirements_38.txt` for python3.8). ## Build with CMake @@ -574,8 +574,9 @@ SymForce into your Python environment, so you'll need to add it to your PYTHONPA Install python requirements: ```bash -pip install -r dev_requirements.txt +pip install -r dev_requirements_38.txt ``` +(or `dev_requirements_39.txt`/`dev_requirements_310.txt` for python3.9 and python3.10, respectively). Build SymForce (requires C++14 or later): ```bash diff --git a/dev_requirements_310.txt b/dev_requirements_310.txt new file mode 100644 index 000000000..abed59497 --- /dev/null +++ b/dev_requirements_310.txt @@ -0,0 +1,372 @@ +# +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: +# +# python test/symforce_requirements_test.py --update +# +--index-url https://pypi.python.org/simple + +alabaster==0.7.12 + # via sphinx +argh==0.26.2 + # via + # skymarshal + # symforce (setup.py) +astroid==2.11.5 + # via pylint +asttokens==2.0.5 + # via stack-data +attrs==21.4.0 + # via jsonschema +babel==2.10.1 + # via sphinx +backcall==0.2.0 + # via ipython +beautifulsoup4==4.11.1 + # via nbconvert +black[jupyter]==21.12b0 + # via symforce (setup.py) +bleach==5.0.0 + # via nbconvert +breathe==4.33.1 + # via symforce (setup.py) +certifi==2022.5.18.1 + # via requests +charset-normalizer==2.0.12 + # via requests +clang-format==14.0.3 + # via symforce (setup.py) +click==8.0.4 + # via + # black + # pip-tools + # symforce (setup.py) +cmake==3.22.4 + # via symforce (setup.py) +coverage==6.4 + # via symforce (setup.py) +cycler==0.11.0 + # via matplotlib +cython==0.29.30 + # via symforce (setup.py) +debugpy==1.6.0 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +dill==0.3.5.1 + # via pylint +docutils==0.17.1 + # via + # breathe + # myst-parser + # nbsphinx + # sphinx +entrypoints==0.4 + # via + # jupyter-client + # nbconvert +executing==0.8.3 + # via stack-data +fastjsonschema==2.15.3 + # via nbformat +fonttools==4.33.3 + # via matplotlib +graphviz==0.20 + # via symforce (setup.py) +idna==3.3 + # via requests +imagesize==1.3.0 + # via sphinx +ipykernel==6.13.0 + # via symforce (setup.py) +ipython==8.4.0 + # via + # black + # ipykernel +ipython-genutils==0.2.0 + # via symforce (setup.py) +isort==5.10.1 + # via + # pylint + # symforce (setup.py) +jedi==0.18.1 + # via ipython +jinja2==3.0.3 + # via + # myst-parser + # nbconvert + # nbsphinx + # skymarshal + # sphinx + # symforce (setup.py) +jsonschema==4.6.0 + # via nbformat +jupyter-client==7.3.1 + # via + # ipykernel + # nbclient +jupyter-core==4.10.0 + # via + # jupyter-client + # nbconvert + # nbformat +jupyterlab-pygments==0.2.2 + # via nbconvert +kiwisolver==1.4.2 + # via matplotlib +lazy-object-proxy==1.7.1 + # via astroid +llvmlite==0.39.0 + # via numba +markdown-it-py==2.1.0 + # via + # mdit-py-plugins + # myst-parser +markupsafe==2.1.1 + # via + # jinja2 + # nbconvert +matplotlib==3.5.2 + # via symforce (setup.py) +matplotlib-inline==0.1.3 + # via + # ipykernel + # ipython +mccabe==0.7.0 + # via pylint +mdit-py-plugins==0.3.0 + # via myst-parser +mdurl==0.1.1 + # via markdown-it-py +mistune==0.8.4 + # via nbconvert +mpmath==1.2.1 + # via sympy +mypy==0.910 + # via symforce (setup.py) +mypy-extensions==0.4.3 + # via + # black + # mypy +myst-parser==0.17.2 + # via symforce (setup.py) +nbclient==0.6.4 + # via nbconvert +nbconvert==6.5.0 + # via nbsphinx +nbformat==5.4.0 + # via + # nbclient + # nbconvert + # nbsphinx + # nbstripout +nbsphinx==0.8.8 + # via symforce (setup.py) +nbstripout==0.5.0 + # via symforce (setup.py) +nest-asyncio==1.5.5 + # via + # ipykernel + # jupyter-client + # nbclient +numba==0.56.0 + # via symforce (setup.py) +numpy==1.22.4 + # via + # matplotlib + # numba + # pandas + # scipy + # skymarshal + # symforce (setup.py) + # symforce-sym +packaging==21.3 + # via + # ipykernel + # matplotlib + # nbconvert + # sphinx +pandas==1.4.2 + # via symforce (setup.py) +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi +pathspec==0.9.0 + # via black +pep517==0.12.0 + # via pip-tools +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pillow==9.1.1 + # via matplotlib +pip-tools==6.6.2 + # via symforce (setup.py) +platformdirs==2.5.2 + # via + # black + # pylint +plotly==5.8.0 + # via symforce (setup.py) +ply==3.11 + # via skymarshal +prompt-toolkit==3.0.29 + # via ipython +psutil==5.9.1 + # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data +pybind11-stubgen==0.10.5 + # via symforce (setup.py) +pygments==2.12.0 + # via + # ipython + # nbconvert + # sphinx +pylint==2.14.0 + # via symforce (setup.py) +pyparsing==3.0.9 + # via + # matplotlib + # packaging +pyrsistent==0.18.1 + # via jsonschema +python-dateutil==2.8.2 + # via + # jupyter-client + # matplotlib + # pandas +pytz==2022.1 + # via + # babel + # pandas +pyyaml==6.0 + # via myst-parser +pyzmq==23.0.0 + # via jupyter-client +requests==2.27.1 + # via sphinx +scipy==1.8.1 + # via symforce (setup.py) +six==1.16.0 + # via + # bleach + # python-dateutil + # skymarshal +file:./third_party/skymarshal + # via symforce (setup.py) +snowballstemmer==2.2.0 + # via sphinx +soupsieve==2.3.2.post1 + # via beautifulsoup4 +sphinx==4.5.0 + # via + # breathe + # myst-parser + # nbsphinx + # sphinx-autodoc-typehints + # symforce (setup.py) +sphinx-autodoc-typehints==1.14.1 + # via symforce (setup.py) +sphinxcontrib-applehelp==1.0.2 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==2.0.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via sphinx +stack-data==0.2.0 + # via ipython +file:./gen/python + # via symforce (setup.py) +sympy==1.10.1 + # via symforce (setup.py) +tenacity==8.0.1 + # via plotly +tinycss2==1.1.1 + # via nbconvert +tokenize-rt==4.2.1 + # via black +toml==0.10.2 + # via mypy +tomli==1.2.3 + # via + # black + # pep517 + # pylint +tomlkit==0.11.0 + # via pylint +tornado==6.1 + # via + # ipykernel + # jupyter-client +traitlets==5.2.2.post1 + # via + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # nbsphinx +types-docutils==0.18.3 + # via types-pygments +types-jinja2==2.11.9 + # via symforce (setup.py) +types-markupsafe==1.1.10 + # via types-jinja2 +types-pygments==2.9.19 + # via symforce (setup.py) +types-requests==2.27.30 + # via symforce (setup.py) +types-setuptools==57.4.17 + # via + # symforce (setup.py) + # types-pygments +types-urllib3==1.26.15 + # via types-requests +typing-extensions==4.2.0 + # via + # black + # mypy + # myst-parser +urllib3==1.26.9 + # via requests +wcwidth==0.2.5 + # via prompt-toolkit +webencodings==0.5.1 + # via + # bleach + # tinycss2 +wheel==0.37.1 + # via + # pip-tools + # symforce (setup.py) +wrapt==1.14.1 + # via astroid + +# The following packages are considered to be unsafe in a requirements file: +pip==22.1.2 + # via + # pip-tools + # symforce (setup.py) +setuptools==62.3.2 + # via + # astroid + # ipython + # numba + # pip-tools + # symforce (setup.py) diff --git a/dev_requirements.txt b/dev_requirements_38.txt similarity index 100% rename from dev_requirements.txt rename to dev_requirements_38.txt diff --git a/dev_requirements_39.txt b/dev_requirements_39.txt new file mode 100644 index 000000000..d73961b17 --- /dev/null +++ b/dev_requirements_39.txt @@ -0,0 +1,378 @@ +# +# This file is autogenerated by pip-compile with python 3.9 +# To update, run: +# +# python test/symforce_requirements_test.py --update +# +--index-url https://pypi.python.org/simple + +alabaster==0.7.12 + # via sphinx +argh==0.26.2 + # via + # skymarshal + # symforce (setup.py) +astroid==2.11.5 + # via pylint +asttokens==2.0.5 + # via stack-data +attrs==21.4.0 + # via jsonschema +babel==2.10.1 + # via sphinx +backcall==0.2.0 + # via ipython +beautifulsoup4==4.11.1 + # via nbconvert +black[jupyter]==21.12b0 + # via symforce (setup.py) +bleach==5.0.0 + # via nbconvert +breathe==4.33.1 + # via symforce (setup.py) +certifi==2022.5.18.1 + # via requests +charset-normalizer==2.0.12 + # via requests +clang-format==14.0.3 + # via symforce (setup.py) +click==8.0.4 + # via + # black + # pip-tools + # symforce (setup.py) +cmake==3.22.4 + # via symforce (setup.py) +coverage==6.4 + # via symforce (setup.py) +cycler==0.11.0 + # via matplotlib +cython==0.29.30 + # via symforce (setup.py) +debugpy==1.6.0 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +dill==0.3.5.1 + # via pylint +docutils==0.17.1 + # via + # breathe + # myst-parser + # nbsphinx + # sphinx +entrypoints==0.4 + # via + # jupyter-client + # nbconvert +executing==0.8.3 + # via stack-data +fastjsonschema==2.15.3 + # via nbformat +fonttools==4.33.3 + # via matplotlib +graphviz==0.20 + # via symforce (setup.py) +idna==3.3 + # via requests +imagesize==1.3.0 + # via sphinx +importlib-metadata==4.11.4 + # via sphinx +ipykernel==6.13.0 + # via symforce (setup.py) +ipython==8.4.0 + # via + # black + # ipykernel +ipython-genutils==0.2.0 + # via symforce (setup.py) +isort==5.10.1 + # via + # pylint + # symforce (setup.py) +jedi==0.18.1 + # via ipython +jinja2==3.0.3 + # via + # myst-parser + # nbconvert + # nbsphinx + # skymarshal + # sphinx + # symforce (setup.py) +jsonschema==4.6.0 + # via nbformat +jupyter-client==7.3.1 + # via + # ipykernel + # nbclient +jupyter-core==4.10.0 + # via + # jupyter-client + # nbconvert + # nbformat +jupyterlab-pygments==0.2.2 + # via nbconvert +kiwisolver==1.4.2 + # via matplotlib +lazy-object-proxy==1.7.1 + # via astroid +llvmlite==0.39.0 + # via numba +markdown-it-py==2.1.0 + # via + # mdit-py-plugins + # myst-parser +markupsafe==2.1.1 + # via + # jinja2 + # nbconvert +matplotlib==3.5.2 + # via symforce (setup.py) +matplotlib-inline==0.1.3 + # via + # ipykernel + # ipython +mccabe==0.7.0 + # via pylint +mdit-py-plugins==0.3.0 + # via myst-parser +mdurl==0.1.1 + # via markdown-it-py +mistune==0.8.4 + # via nbconvert +mpmath==1.2.1 + # via sympy +mypy==0.910 + # via symforce (setup.py) +mypy-extensions==0.4.3 + # via + # black + # mypy +myst-parser==0.17.2 + # via symforce (setup.py) +nbclient==0.6.4 + # via nbconvert +nbconvert==6.5.0 + # via nbsphinx +nbformat==5.4.0 + # via + # nbclient + # nbconvert + # nbsphinx + # nbstripout +nbsphinx==0.8.8 + # via symforce (setup.py) +nbstripout==0.5.0 + # via symforce (setup.py) +nest-asyncio==1.5.5 + # via + # ipykernel + # jupyter-client + # nbclient +numba==0.56.0 + # via symforce (setup.py) +numpy==1.22.4 + # via + # matplotlib + # numba + # pandas + # scipy + # skymarshal + # symforce (setup.py) + # symforce-sym +packaging==21.3 + # via + # ipykernel + # matplotlib + # nbconvert + # sphinx +pandas==1.4.2 + # via symforce (setup.py) +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi +pathspec==0.9.0 + # via black +pep517==0.12.0 + # via pip-tools +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pillow==9.1.1 + # via matplotlib +pip-tools==6.6.2 + # via symforce (setup.py) +platformdirs==2.5.2 + # via + # black + # pylint +plotly==5.8.0 + # via symforce (setup.py) +ply==3.11 + # via skymarshal +prompt-toolkit==3.0.29 + # via ipython +psutil==5.9.1 + # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data +pybind11-stubgen==0.10.5 + # via symforce (setup.py) +pygments==2.12.0 + # via + # ipython + # nbconvert + # sphinx +pylint==2.14.0 + # via symforce (setup.py) +pyparsing==3.0.9 + # via + # matplotlib + # packaging +pyrsistent==0.18.1 + # via jsonschema +python-dateutil==2.8.2 + # via + # jupyter-client + # matplotlib + # pandas +pytz==2022.1 + # via + # babel + # pandas +pyyaml==6.0 + # via myst-parser +pyzmq==23.0.0 + # via jupyter-client +requests==2.27.1 + # via sphinx +scipy==1.8.1 + # via symforce (setup.py) +six==1.16.0 + # via + # bleach + # python-dateutil + # skymarshal +file:./third_party/skymarshal + # via symforce (setup.py) +snowballstemmer==2.2.0 + # via sphinx +soupsieve==2.3.2.post1 + # via beautifulsoup4 +sphinx==4.5.0 + # via + # breathe + # myst-parser + # nbsphinx + # sphinx-autodoc-typehints + # symforce (setup.py) +sphinx-autodoc-typehints==1.14.1 + # via symforce (setup.py) +sphinxcontrib-applehelp==1.0.2 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==2.0.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via sphinx +stack-data==0.2.0 + # via ipython +file:./gen/python + # via symforce (setup.py) +sympy==1.10.1 + # via symforce (setup.py) +tenacity==8.0.1 + # via plotly +tinycss2==1.1.1 + # via nbconvert +tokenize-rt==4.2.1 + # via black +toml==0.10.2 + # via mypy +tomli==1.2.3 + # via + # black + # pep517 + # pylint +tomlkit==0.11.0 + # via pylint +tornado==6.1 + # via + # ipykernel + # jupyter-client +traitlets==5.2.2.post1 + # via + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # nbsphinx +types-docutils==0.18.3 + # via types-pygments +types-jinja2==2.11.9 + # via symforce (setup.py) +types-markupsafe==1.1.10 + # via types-jinja2 +types-pygments==2.9.19 + # via symforce (setup.py) +types-requests==2.27.30 + # via symforce (setup.py) +types-setuptools==57.4.17 + # via + # symforce (setup.py) + # types-pygments +types-urllib3==1.26.15 + # via types-requests +typing-extensions==4.2.0 + # via + # astroid + # black + # mypy + # myst-parser + # pylint +urllib3==1.26.9 + # via requests +wcwidth==0.2.5 + # via prompt-toolkit +webencodings==0.5.1 + # via + # bleach + # tinycss2 +wheel==0.37.1 + # via + # pip-tools + # symforce (setup.py) +wrapt==1.14.1 + # via astroid +zipp==3.8.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +pip==22.1.2 + # via + # pip-tools + # symforce (setup.py) +setuptools==62.3.2 + # via + # astroid + # ipython + # numba + # pip-tools + # symforce (setup.py) diff --git a/test/symforce_requirements_test.py b/test/symforce_requirements_test.py index 19c686cdb..d7b40cffb 100644 --- a/test/symforce_requirements_test.py +++ b/test/symforce_requirements_test.py @@ -36,8 +36,9 @@ def main(*args: T.Any, **kwargs: T.Any) -> None: def test_requirements(self) -> None: output_dir = Path(self.make_output_dir("sf_requirements_test_")) - output_requirements_file = output_dir / "dev_requirements.txt" - symforce_requirements_file = SYMFORCE_DIR / "dev_requirements.txt" + requirements_name = f"dev_requirements_{sys.version_info.major}{sys.version_info.minor}.txt" + output_requirements_file = output_dir / requirements_name + symforce_requirements_file = SYMFORCE_DIR / requirements_name local_requirements_map = { f"skymarshal @ file://localhost/{SYMFORCE_DIR}/third_party/skymarshal": "file:./third_party/skymarshal", @@ -76,7 +77,7 @@ def test_requirements(self) -> None: cwd=SYMFORCE_DIR, env=dict( os.environ, - # Compile command to put in the header of dev_requirements.txt + # Compile command to put in the header of dev_requirements_XY.txt CUSTOM_COMPILE_COMMAND="python test/symforce_requirements_test.py --update", ), )