Merge pull request #219 from amarquand/dev

Address multiple issues

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
 
     - name: Run Tests with coverage and save report
       run: |
-        coverage run -m pytest pytest/ --junitxml=report.xml --html=report.html
+        coverage run -m pytest tests/ --junitxml=report.xml --html=report.html
         coverage xml -o coverage.xml
       continue-on-error: true  # Continue to the next step even if tests fail
 

.gitignore

@@ -83,3 +83,17 @@ dist/pcntoolkit-0.27-py3.11.egg
 # Basic test functions for SHASH
 tests/test_SHASH.ipynb
 
+# Dist folder
+dist/pcntoolkit-0.30.post2-py3.12.egg
+dist/*
+dist
+build/*
+
+# CLI test folder
+tests/cli_test/*
+!tests/cli_test/test_cli.sh
+!tests/cli_test/split_data.py
+
+docs/autoapi/*
+docs/_build/*
+docs

CHANGES

@@ -87,3 +87,18 @@ version 0.29
 - Addedd functionality to compute SHASH z-scores from normative.py
 - Updated requirements
 - Basic pytest continuous integration framework implemented
+
+version 0.30.0
+- Minor bug fixes
+
+version 0.31.0
+- Major changes:
+    - Move to Poetry for dependency management in pyproject.toml.
+    - PCNToolkit must now be installed using python -m pip install .. See the README for complete instructions.
+    - A CLI command normative is automatically created, and can be used instead of python normative.py.
+    - Nutpie can be used as a sampler for HBR by setting `nuts_sampler='nutpie''. Nutpie and numba must first be installed using conda.
+- Minor changes
+    - torque jobs now support multicore jobs via the keyword 'n_cores_per_batch'
+    - Backwards compatibilty improved by using pd.read_pickle instead of pickle.load
+    - SHASH classes have been refactored and improved
+    - HBR priors improved

README.md

@@ -8,64 +8,94 @@ Methods for normative modelling, spatial statistics and pattern recognition. Doc
 
 ## Basic installation (on a local machine)
 
-i) install anaconda3 ii) create enviornment with "conda create --name <env_name>" iii) activate environment by "source activate <env_name>" iv) install required conda packages
+#### Install anaconda3 
+
+using the download here: https://www.anaconda.com/download
+
+#### Create environment 
+```
+conda create -n <env_name> python==3.12
+```
+
+#### Activate environment
 
 ```
-conda install pip pandas scipy
+source activate <env_name>
 ```
 
-v) install PCNtoolkit (plus dependencies)
+### Install nutpie and numba from conda-forge
 
+```
+conda install nutpie numba -c conda-forge
+```
+
+#### Install torch from pytorch.org
+
+Use the command that you get from the command builder here: https://pytorch.org/get-started/locally/. This will ensure you do not install the CUDA version of torch if your pc does not have a GPU. We also recommend that you use the `conda` option. 
+
+
+#### Install PCNtoolkit
+
+Using pip:
 ```
 pip install pcntoolkit
 ```
 
+Using a local clone of the repo:
+```
+python -m pip install .
+```
+
 ## Alternative installation (on a shared resource)
-Make sure conda is available on the system.
+
+#### Make sure conda is available on the system.
 Otherwise install it first from https://www.anaconda.com/ 
 
 ```
 conda --version
 ```
 
-Create a conda environment in a shared location
+#### Create a conda environment in a shared location
 
 ```
-conda create -y python==3.8.3 numpy mkl blas --prefix=/shared/conda/<env_name>
+conda create -y python==3.12 numpy mkl blas --prefix=/shared/conda/<env_name>
 ```
 
-Activate the conda environment 
+#### Activate the conda environment 
 
 ```
 conda activate /shared/conda/<env_name>
 ```
 
-Install other dependencies
+### Install nutpie using conda 
 
 ```
-conda install -y pandas scipy 
+conda install nutpie numba -c conda-forge
 ```
 
-Install pip dependencies
+#### install torch 
+
+Using the command that you get from the command builder here:
 
 ```
-pip --no-cache-dir install nibabel scikit-learn torch glob3 
+https://pytorch.org/get-started/locally/
 ```
 
-Clone the repo
+If your shared resource has no GPU, make sure you select the 'CPU' field in the 'Compute Platform' row. Here we also prefer conda over pip.
+
+#### Clone the repo
 
 ```
 git clone https://github.com/amarquand/PCNtoolkit.git
 ```
 
-install in the conda environment
+### Install in the conda environment
 
 ```
 cd PCNtoolkit/
-python3 setup.py install
+python -m pip install .
 ```
-
-Test 
+### Test 
 ```
 python -c "import pcntoolkit as pk;print(pk.__file__)"
 ```

doc/build/html/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
-# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 1b4d3aa4bec5023b71b7824b4d305268
+# This file records the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: d51dcb6ba2cb6f1cbbeaf2b9ee091add
 tags: 645f666f9bcd5a90fca523b33c5a78b7

doc/build/html/_modules/bayesreg.html

@@ -1,28 +1,24 @@
+
+
 <!DOCTYPE html>
-<html class="writer-html5" lang="en">
+<html class="writer-html5" lang="en" data-content_root="../">
 <head>
   <meta charset="utf-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
   <title>bayesreg &mdash; Predictive Clinical Neuroscience Toolkit 0.20 documentation</title>
-      <link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
-      <link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
-      <link rel="stylesheet" type="text/css" href="../_static/graphviz.css" />
+      <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=fa44fd50" />
+      <link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
+      <link rel="stylesheet" type="text/css" href="../_static/graphviz.css?v=4ae1632d" />
       <link rel="stylesheet" type="text/css" href="../_static/pages/css/pcntoolkit_tabs.css" />
-      <link rel="stylesheet" type="text/css" href="../_static/tabs.css" />
       <link rel="stylesheet" type="text/css" href="../_static/pages/css/pcntoolkit.css" />
       <link rel="stylesheet" type="text/css" href="../_static/pages/css/pcntoolkit_nomaxwidth.css" />
 
 
-  <!--[if lt IE 9]>
-    <script src="../_static/js/html5shiv.min.js"></script>
-  <![endif]-->
-
-        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
-        <script src="../_static/jquery.js"></script>
-        <script src="../_static/underscore.js"></script>
-        <script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
-        <script src="../_static/doctools.js"></script>
-        <script src="../_static/tabs.js"></script>
+      <script src="../_static/jquery.js?v=5d32c60e"></script>
+      <script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+      <script src="../_static/documentation_options.js?v=0144e382"></script>
+      <script src="../_static/doctools.js?v=9bcbadda"></script>
+      <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="../_static/js/theme.js"></script>
     <link rel="index" title="Index" href="../genindex.html" />
     <link rel="search" title="Search" href="../search.html" /> 
@@ -62,13 +58,24 @@
 </ul>
 <p class="caption" role="heading"><span class="caption-text">Tutorials</span></p>
 <ul>
-<li class="toctree-l1"><a class="reference internal" href="../pages/normative_modelling_walkthrough.html">Gaussian Process Regression</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../pages/HBR_NormativeModel_FCONdata_Tutorial.html">Hierarchical Bayesian Regression</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/normative_modelling_walkthrough.html"><strong>DEMO ON NORMATIVE MODELING</strong></a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/HBR_NormativeModel_FCONdata_Tutorial.html">Predictive Clinical Neuroscience Toolkit</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/HBR_NormativeModel_FCONdata_Tutorial.html#hierarchical-bayesian-regression-normative-modelling-and-transfer-onto-unseen-site">Hierarchical Bayesian Regression Normative Modelling and Transfer onto unseen site.</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../pages/apply_normative_models.html">Braincharts: transfer</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../pages/BLR_normativemodel_protocol.html">Bayesian Linear Regression</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/BLR_normativemodel_protocol.html">Predictive Clinical Neuroscience Toolkit</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/BLR_normativemodel_protocol.html#the-normative-modeling-framework-for-computational-psychiatry-protocol">The Normative Modeling Framework for Computational Psychiatry Protocol</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/BLR_normativemodel_protocol.html#data-preparation">Data Preparation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/BLR_normativemodel_protocol.html#algorithm-modeling">Algorithm &amp; Modeling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/BLR_normativemodel_protocol.html#evaluation-interpretation">Evaluation &amp; Interpretation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../pages/visualizations.html">Visualization of normative modeling outputs</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../pages/post_hoc_analysis.html">Post-hoc analysis on normative modeling outputs</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../pages/other_predictive_models.html">Predictive modeling using deviation scores</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/other_predictive_models.html">Load Data</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/other_predictive_models.html#create-train-test-splits">Create Train/Test Splits</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/other_predictive_models.html#principal-component-regression-bbs">Principal Component Regression (BBS)</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/other_predictive_models.html#connectome-predictive-modelling">Connectome Predictive Modelling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/other_predictive_models.html#lasso-linear-regression-l1-regularization">Lasso (Linear Regression + L1 Regularization)</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/other_predictive_models.html#ridge-linear-regression-l2-regularization">Ridge (Linear Regression + L2 Regularization)</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../pages/other_predictive_models.html#elastic-net-linear-regression-l1-l2-regularization">Elastic Net (Linear Regression + L1/L2 Regularization)</a></li>
 </ul>
 <p class="caption" role="heading"><span class="caption-text">Other Useful Stuff</span></p>
 <ul>
@@ -111,7 +118,9 @@ <h1>Source code for bayesreg</h1><div class="highlight"><pre>
 from scipy.linalg import LinAlgError
 
 
-<div class="viewcode-block" id="BLR"><a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR">[docs]</a>class BLR:
+<div class="viewcode-block" id="BLR">
+<a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR">[docs]</a>
+class BLR:
     &quot;&quot;&quot;Bayesian linear regression
 
     Estimation and prediction of Bayesian linear regression models
@@ -253,7 +262,9 @@ <h1>Source code for bayesreg</h1><div class="highlight"><pre>
 
         return beta, alpha, gamma
 
-<div class="viewcode-block" id="BLR.post"><a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.post">[docs]</a>    def post(self, hyp, X, y, Xv=None):
+<div class="viewcode-block" id="BLR.post">
+<a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.post">[docs]</a>
+    def post(self, hyp, X, y, Xv=None):
         &quot;&quot;&quot; Generic function to compute posterior distribution.
 
             This function will save the posterior mean and precision matrix as
@@ -306,7 +317,10 @@ <h1>Source code for bayesreg</h1><div class="highlight"><pre>
         self.D = D
         self.hyp = hyp</div>
 
-<div class="viewcode-block" id="BLR.loglik"><a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.loglik">[docs]</a>    def loglik(self, hyp, X, y, Xv=None):
+
+<div class="viewcode-block" id="BLR.loglik">
+<a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.loglik">[docs]</a>
+    def loglik(self, hyp, X, y, Xv=None):
         &quot;&quot;&quot; Function to compute compute log (marginal) likelihood &quot;&quot;&quot;
 
         # hyperparameters (alpha not needed)
@@ -363,7 +377,10 @@ <h1>Source code for bayesreg</h1><div class="highlight"><pre>
         self.nlZ = nlZ
         return nlZ</div>
 
-<div class="viewcode-block" id="BLR.penalized_loglik"><a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.penalized_loglik">[docs]</a>    def penalized_loglik(self, hyp, X, y, Xv=None, l=0.1, norm=&#39;L1&#39;):
+
+<div class="viewcode-block" id="BLR.penalized_loglik">
+<a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.penalized_loglik">[docs]</a>
+    def penalized_loglik(self, hyp, X, y, Xv=None, l=0.1, norm=&#39;L1&#39;):
         &quot;&quot;&quot; Function to compute the penalized log (marginal) likelihood 
 
             :param hyp: hyperparameter vector
@@ -382,7 +399,10 @@ <h1>Source code for bayesreg</h1><div class="highlight"><pre>
             print(&quot;Requested penalty not recognized, choose between &#39;L1&#39; or &#39;L2&#39;.&quot;)
         return L</div>
 
-<div class="viewcode-block" id="BLR.dloglik"><a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.dloglik">[docs]</a>    def dloglik(self, hyp, X, y, Xv=None):
+
+<div class="viewcode-block" id="BLR.dloglik">
+<a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.dloglik">[docs]</a>
+    def dloglik(self, hyp, X, y, Xv=None):
         &quot;&quot;&quot; Function to compute derivatives &quot;&quot;&quot;
 
         # hyperparameters
@@ -484,8 +504,11 @@ <h1>Source code for bayesreg</h1><div class="highlight"><pre>
         self.dnlZ = dnlZ
         return dnlZ</div>
 
+
     # model estimation (optimization)
-<div class="viewcode-block" id="BLR.estimate"><a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.estimate">[docs]</a>    def estimate(self, hyp0, X, y, **kwargs):
+<div class="viewcode-block" id="BLR.estimate">
+<a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.estimate">[docs]</a>
+    def estimate(self, hyp0, X, y, **kwargs):
         &quot;&quot;&quot; Function to estimate the model 
 
             :param hyp: hyperparameter vector
@@ -542,7 +565,10 @@ <h1>Source code for bayesreg</h1><div class="highlight"><pre>
 
         return self.hyp</div>
 
-<div class="viewcode-block" id="BLR.predict"><a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.predict">[docs]</a>    def predict(self, hyp, X, y, Xs,
+
+<div class="viewcode-block" id="BLR.predict">
+<a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.predict">[docs]</a>
+    def predict(self, hyp, X, y, Xs,
                 var_groups_test=None,
                 var_covariates_test=None, **kwargs):
         &quot;&quot;&quot; Function to make predictions from the model
@@ -595,7 +621,10 @@ <h1>Source code for bayesreg</h1><div class="highlight"><pre>
 
         return ys, s2</div>
 
-<div class="viewcode-block" id="BLR.predict_and_adjust"><a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.predict_and_adjust">[docs]</a>    def predict_and_adjust(self, hyp, X, y, Xs=None,
+
+<div class="viewcode-block" id="BLR.predict_and_adjust">
+<a class="viewcode-back" href="../pages/modindex.html#bayesreg.BLR.predict_and_adjust">[docs]</a>
+    def predict_and_adjust(self, hyp, X, y, Xs=None,
                            ys=None,
                            var_groups_test=None,
                            var_groups_adapt=None, **kwargs):
@@ -687,7 +716,9 @@ <h1>Source code for bayesreg</h1><div class="highlight"><pre>
                     ys = ys - residuals_mu
                 s2_out = None
 
-        return ys_out, s2_out</div></div>
+        return ys_out, s2_out</div>
+</div>
+
 </pre></div>
 
            </div>