Use Poetry for installing and package maintenance

.gitignore

@@ -101,9 +101,9 @@ venv.bak/
 # mypy
 .mypy_cache/
 
-# egyéb
+# Others (project specific)
 warcs
 logs
 
 #PyCharm
-.idea
+.idea/

.travis.yml

@@ -1,9 +1,39 @@
 language: python
-dist: bionic
-sudo: true
+os: linux
+dist: focal
 python:
-  - '3.7'
+  - '3.8'
+before_install:
+  - pip install poetry
 install:
-  - make venv install
+  # Install packages with APT
+  # - make install-dep-packages
+  # When cython is needed to be installed separately
+  # - pip install cython
+  - make venv
+  # - make install-dep-packages
 script:
-  - make test
+  - make install test
+deploy:
+  - provider: releases
+    skip_cleanup: true
+    # Create a personal access token and store it in travis-ci as an environment variable
+    api_key: $GITHUB_TOKEN
+    file_glob: true
+    file: dist/*
+    on:
+      repo: ELTE-DH/WebArticleCurator
+      branch: master
+      tags: true
+      condition: $TRAVIS_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+$
+  - provider: script
+    skip_cleanup: true
+    # Upload to PyPi with poetry (create a token and store it in travis-ci as PYPI_TOKEN environment variable)
+    script: make upload
+    on:
+      repo: ELTE-DH/WebArticleCurator
+      branch: master
+      tags: true
+      condition: $TRAVIS_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+$
+notifications:
+    email: false

Makefile

@@ -1,41 +1,111 @@
 .DEFAULT_GOAL = all
+# Bash is needed for time, compgen, [[ and other builtin commands
+SHELL := /bin/bash -o pipefail
 RED := $(shell tput setaf 1)
 GREEN := $(shell tput setaf 2)
 NOCOLOR := $(shell tput sgr0)
 PYTHON := python3
-VENVDIR := $(CURDIR)/venv
-VENVPIP := $(VENVDIR)/bin/python -m pip
-VENVPYTHON := $(VENVDIR)/bin/python
 
-all:
-	@echo "See Makefile for possible targets!"
+# Module specific parameters
+MODULE := webarticlecurator
+
+# These targets do not show as possible target with bash completion
+__extra-deps:
+	@# Do extra stuff (e.g. compiling, downloading) before building the package
+	@exit 0
+.PHONY: __extra-deps
+
+__clean-extra-deps:
+	@# e.g. @rm -rf stuff
+	@exit 0
+.PHONY: __clean-extra-deps
+
+# From here only generic parts
+
+all: clean venv install test
+	@echo "all: clean, venv, install, test"
+.PHONY: all
+
+install-dep-packages:
+	@echo "Installing needed packages from Aptfile..."
+	@command -v apt-get >/dev/null 2>&1 || \
+			(echo >&2 "$(RED)Command 'apt-get' could not be found!$(NOCOLOR)"; exit 1)
+	@# Aptfile can be omited if empty
+	@[[ ! -f "$(CURDIR)/Aptfile" ]] || \
+	    ([[ $$(dpkg -l | grep -wcf $(CURDIR)/Aptfile) -eq $$(cat $(CURDIR)/Aptfile | wc -l) ]] || \
+		(sudo -E apt-get update && \
+		sudo -E apt-get -yq --no-install-suggests --no-install-recommends $(travis_apt_get_options) install \
+			`cat $(CURDIR)/Aptfile`))
+	@echo "$(GREEN)Needed packages are succesfully installed!$(NOCOLOR)"
+.PHONY: install-dep-packages
 
 venv:
-	@echo "Creating virtualenv in $(VENVDIR)...$(NOCOLOR)"
-	@rm -rf $(VENVDIR)
-	@$(PYTHON) -m venv $(VENVDIR)
-	@$(VENVPIP) install wheel twine
-	@$(VENVPIP) install -r requirements.txt
-	@echo "$(GREEN)Virtualenv is successfully created!$(NOCOLOR)"
+	@poetry install --no-root
 .PHONY: venv
 
-build:
-	$(VENVPYTHON) setup.py sdist bdist_wheel
+build: install-dep-packages venv __extra-deps
+	@poetry build
+.PHONY: build
 
+# Install the actual wheel package to test it in later steps
 install: build
-	@echo "Installing package to user..."
-	$(VENVPIP) install --upgrade dist/*.whl
-	@echo "$(GREEN)Package is successfully installed!$(NOCOLOR)"
+	 @poetry run pip install --upgrade dist/*.whl
 .PHONY: install
 
+# Upload to PyPi with poetry (with token if $$PYPI_TOKEN is specified)
 upload:
-	$(VENVPYTHON) -m twine upload dist/*
-
-clean:
-	rm -rf dist/ build/ webarticlecurator.egg-info/
+	@[[ ! -z "$(PYPI_TOKEN)" ]] && poetry publish --username "__token__" --password $(PYPI_TOKEN) || poetry publish
+.PHONY: upload
 
 test:
-	for i in configs/config_*.yaml; do echo "Testing $${i}:"; $(VENVPYTHON) webarticlecurator/utils.py $${i} \
-	 || exit 1; done
-	for i in configs/extractors/site_specific_*.py; do $(VENVPYTHON) $${i} || exit 1; done
+	@# TODO Implement this properly!
+	@# @poetry run python -m pytest --verbose tests/
+	@for i in configs/config_*.yaml; do echo "Testing $${i}:"; $$(poetry env info -p)/bin/python \
+		src/webarticlecurator/utils.py $${i} || exit 1; done
+	for i in configs/extractors/site_specific_*.py; do $$(poetry env info -p)/bin/python $${i} || exit 1; done
 	@echo "$(GREEN)All tests are successfully passed!$(NOCOLOR)"
+
+clean: __clean-extra-deps
+	@rm -rf dist/ .pytest_cache/ $$(poetry env info -p)
+.PHONY: clean
+
+# Do actual release with new version. Originally from: https://github.com/mittelholcz/contextfun
+release-major:
+	@make -s __release BUMP="major"
+.PHONY: release-major
+
+release-minor:
+	@make -s __release BUMP="minor"
+.PHONY: release-minor
+
+release-patch:
+	@make -s __release BUMP="patch"
+.PHONY: release-patch
+
+__release:
+	@[[ "$(BUMP)" == "major" || "$(BUMP)" == "minor" || "$(BUMP)" == "patch" ]] || \
+		(echo -e "$(RED)Do not call this target!\nUse 'release-major', 'release-minor' or 'release-patch'!$(NOCOLOR)"; \
+		 exit 1)
+	@[[ -z $$(git status --porcelain) ]] || (echo "$(RED)Working dir is dirty!$(NOCOLOR)"; exit 1)
+	@# Update dependencies before buiding and testing (closest to clean install)
+	@poetry update
+	@# poetry version will modify pyproject.toml only. The other steps must be done manually.
+	@poetry version $(BUMP)
+	@# Add modified files to git before commit
+	@git add pyproject.toml poetry.lock
+	@# Clean install with (built package) and test
+	@make all
+	@# Create release commit and git tag
+	@make -S __commit_to_origin NEWVER=$$(poetry run python src/$(MODULE)/version.py)
+.PHONY: __release
+
+__commit_to_origin:
+	@[[ ! -z "$(NEWVER)" ]] || \
+		(echo -e "$(RED)Do not call this target!\nUse 'release-major', 'release-minor' or 'release-patch'!$(NOCOLOR)"; \
+		 exit 1)
+	@echo "NEW VERSION: $(NEWVER)"
+	@git commit -m "Release $(NEWVER)"
+	@git tag -a "v$(NEWVER)" -m "Release v$(NEWVER)"
+	@git push
+	@git push --tags
+.PHONY: __commit_to_origin

README.md

@@ -4,7 +4,7 @@ A crawler program which can be used for downloading the content of portals (news
 
 ## Requirements
 
-- Python 3.7+
+- Python 3.8+
 - (optional for corpus converter if installed as `webarticlecurator[newspaper3k]`) for Newspaper3k, the installation of the following packages must precede the installation of this program: python3-dev libxml2-dev libxslt-dev libjpeg-dev zlib1g-dev libpng12-dev
 
 ## Install
@@ -13,10 +13,27 @@ A crawler program which can be used for downloading the content of portals (news
 
 `pip3 install webarticlecurator`
 
+The following extras can be installed:
+
+- Newspaper3k: `newspaper`
+- All the above: `full`
+
+E.g. `pip3 install webarticlecurator[full]`
+
 ### Manual
 
+[_Poetry_](https://python-poetry.org/) and (optionally) [_GNU Make_](https://www.gnu.org/software/make/) are reqired.
+
 1. `git clone https://github.com/ELTE-DH/webarticlecurator.git`
-2. Run `python3 setup.py install` (you may have to use `sudo` at the beginning of this command)
+2. Run `make`
+
+On Windows or without Make (after cloning the repository):
+
+1. `poetry install --no-root`
+2. `poetry build`
+3. `poetry run pip install --upgrade dist/*.whl` (the correct filename must be specified on Windows)
+
+To install extras run: `poetry install -E [NAME OF THE EXTRA TO INSTALL]`
 
 ## Usage