Merge pull request #215 from MansMeg/feature/gh_db

add github database for Python

.github/workflows/workflow-python.yml

@@ -3,9 +3,9 @@ name: Python
 on:
   push:
     branches:
-      - '*'
+      - '**'
     tags:
-      - '*'
+      - '**'
 
 jobs:
   models:
@@ -16,6 +16,8 @@ jobs:
           os: [ubuntu-latest, macos-latest, windows-latest]
           python-version: [3.6, 3.7, 3.8, 3.9]
       fail-fast: false
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Checkout github
         uses: actions/checkout@v2

.github/workflows/workflow-r.yml

@@ -3,9 +3,9 @@ name: R
 on:
   push:
     branches:
-      - '*'
+      - '**'
     tags:
-      - '*'
+      - '**'
 
 jobs:
   models:

python/README.md

@@ -4,12 +4,19 @@ Currently only python 3.6+ is supported. Python 3.5+ support can be added if nee
 
 ## Installation
 
-Currently only local install is supported. From the main directory of this project run
+Installation from PyPI is recommended.
+
 ```bash
-pip install python/
+pip install posteriordb
 ```
 
-Installing from git url will be supported soon. Publishing the package to PyPI will also happen at some point.
+Installing from the local clone.
+
+```bash
+git clone https://github.com/MansMeg/posteriordb
+cd posteriordb
+pip install python/
+```
 
 ## Using the posterior database from python
 
@@ -23,9 +30,28 @@ First we create the posterior database to use, here the cloned posterior databas
 >>> pdb_path = os.path.join(os.getcwd(), "posterior_database")
 >>> my_pdb = PosteriorDatabase(pdb_path)
 ```
-
 The above code requires that your working directory is in the main folder of your copy
-of this project. Alternatively, you can specify the path to the folder directly. To list the posteriors available, use `posterior_names`.
+of this project. Alternatively, you can specify the path to the folder directly.
+
+Online database can be used with the `PosteriorDatabaseGithub` class. Remember to create and set `GITHUB_PAT` environmental variable.
+It's recommended that users create a read-only (no extra permissions) [GitHub Personal Access Token (PAT)](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token) for `posteriordb` use. It's also recommended that the
+`GITHUB_PAT` environmental variable is added to user environmental variables and it is not shown in Python script as in the example below.
+
+
+If not explicitly defined, `PosteriorDatabase` and `PosteriorDatabaseGithub` will create a new (or use old database) located at `POSTERIOR_DB_PATH` if it's
+defined. `PosteriorDatabaseGithub` will finally use `$HOME/.posteriordb/posterior_database` as a fallback location if no environmental variables have been set.
+Each model and data is only downloaded and cached when needed.
+
+```python
+>>> from posteriordb import PosteriorDatabaseGithub
+>>> import os
+>>> # It is recommended that GITHUB_PAT is added to the user environmental variables
+>>> # outside python and not in a python script as shown in this example code
+>>> os.environ["GITHUB_PAT"] = "token-string-here"
+>>> my_pdb = PosteriorDatabaseGithub()
+```
+
+To list the posteriors available, use `posterior_names`.
 
 ```python
 >>> pos = my_pdb.posterior_names()

python/setup.py

@@ -4,6 +4,13 @@
 
 PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__))
 VERSION_FILE = os.path.join(PROJECT_ROOT, "src", "posteriordb", "__init__.py")
+README_FILE = os.path.join(PROJECT_ROOT, "README.md")
+
+
+def get_long_description():
+    with open(README_FILE, "rt") as buff:
+        return buff.read()
+
 
 def get_version():
     lines = open(VERSION_FILE, "rt").readlines()
@@ -14,6 +21,7 @@ def get_version():
             return mo.group(1)
     raise RuntimeError("Unable to find version in %s." % (VERSION_FILE,))
 
+
 setup(
     name="posteriordb",
     version=get_version(),
@@ -22,8 +30,11 @@ def get_version():
     author="Eero Linna",
     author_email="[email protected]",
     license="GPL",
+    long_description=get_long_description(),
+    long_description_content_type="text/markdown",
     packages=["posteriordb"],
     package_dir={"": "src"},
+    install_requires=["requests"],
     zip_safe=False,
     include_package_data=True,
 )

python/src/posteriordb/__init__.py

@@ -1,3 +1,4 @@
 from .posterior_database import PosteriorDatabase
+from .posterior_database_github import PosteriorDatabaseGithub
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"

python/src/posteriordb/dataset.py

@@ -1,14 +1,18 @@
 import json
 import os
 import tempfile
+from typing import Union
 from zipfile import ZipFile
 
 from .posterior_database import PosteriorDatabase
+from .posterior_database_github import PosteriorDatabaseGithub
 from .util import drop_keys
 
 
 class Dataset:
-    def __init__(self, name: str, posterior_db: PosteriorDatabase):
+    def __init__(
+        self, name: str, posterior_db: Union[PosteriorDatabase, PosteriorDatabaseGithub]
+    ):
         self.name = name
         self.posterior_db = posterior_db
         full_information = self.posterior_db.get_data_info(name=self.name)

python/src/posteriordb/model.py

@@ -1,6 +1,8 @@
 import os
+from typing import Union
 
 from .posterior_database import PosteriorDatabase
+from .posterior_database_github import PosteriorDatabaseGithub
 from .pymc3_model_implementation import PyMC3ModelImplementation
 from .stan_model_implementation import StanModelImplementation
 from .util import drop_keys
@@ -9,7 +11,9 @@
 
 
 class Model:
-    def __init__(self, name: str, posterior_db: PosteriorDatabase):
+    def __init__(
+        self, name: str, posterior_db: Union[PosteriorDatabase, PosteriorDatabaseGithub]
+    ):
         self.name = name
         self.posterior_db = posterior_db
         full_model_info = self.posterior_db.get_model_info(name=self.name)

python/src/posteriordb/posterior.py

@@ -1,15 +1,18 @@
 import json
+from typing import Union
 from zipfile import ZipFile
 
 from .dataset import Dataset
 from .model import Model
-from .posterior_database import load_json_file
 from .posterior_database import PosteriorDatabase
+from .posterior_database_github import PosteriorDatabaseGithub
 from .util import drop_keys
 
 
 class Posterior:
-    def __init__(self, name: str, posterior_db: PosteriorDatabase):
+    def __init__(
+        self, name: str, posterior_db: Union[PosteriorDatabase, PosteriorDatabaseGithub]
+    ):
         self.name = name
 
         assert name in posterior_db.posterior_names()

python/src/posteriordb/posterior_database.py

@@ -32,8 +32,10 @@ def filenames_in_dir_no_extension(directory, extension):
 
 
 class PosteriorDatabase:
-    def __init__(self, path: str):
-        self.path = path
+    def __init__(self, path: str = None):
+        if path is None:
+            path = os.environ.get("POSTERIOR_DB_PATH")
+        self.path = str(path)
         # TODO assert that path is a valid posterior database
 
     def full_path(self, path: str):