diff --git a/.env.example b/.env.example index d4838f4..4891501 100644 --- a/.env.example +++ b/.env.example @@ -20,3 +20,5 @@ DJANGO_SUPERUSER_PASSWORD='admin' CACHE_LOCATION='memcached:11211' CACHE_USERNAME= CACHE_PASSWORD= +BROKER_URL='amqp://guest:guest@172.17.0.1:5672//' +RESULT_BACKEND_URL='rpc://' diff --git a/Procfile b/Procfile index 18bc0bd..8fb0eae 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,4 @@ release: python manage.py migrate web: gunicorn config.wsgi --log-file - +worker: celery -A config worker -l info --concurrency=4 +beat: celery -A config beat -l info diff --git a/config/__init__.py b/config/__init__.py index e69de29..53f4ccb 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/config/celery.py b/config/celery.py new file mode 100644 index 0000000..ea6f696 --- /dev/null +++ b/config/celery.py @@ -0,0 +1,19 @@ +import os + +from celery import Celery + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") + +app = Celery("config") + +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Auto-discover tasks in all installed apps +# This will look for a 'tasks.py' file in each app directory +app.autodiscover_tasks() + + +@app.task(bind=True) +def debug_task(self): + """A simple task for testing Celery setup""" + print(f"Request: {self.request!r}") diff --git a/config/settings.py b/config/settings.py index c2388c3..c2d876b 100644 --- a/config/settings.py +++ b/config/settings.py @@ -4,10 +4,14 @@ import dj_database_url import sentry_sdk +from celery.schedules import crontab from sentry_sdk.integrations.django import DjangoIntegration +from dotenv import load_dotenv from contributors.utils import misc +load_dotenv() + BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Logging setup @@ -291,3 +295,19 @@ } SITE_ID = 1 + +# Just broker URL, no result backend needed +CELERY_BROKER_URL = os.getenv('BROKER_URL', 'amqp://guest:guest@172.17.0.1:5672//') +CELERY_RESULT_BACKEND = None +CELERY_IGNORE_RESULT = True + +CELERY_BEAT_SCHEDULE = { + 'sync-github-repositories': { + 'task': 'contributors.tasks.sync_github_data', + 'schedule': crontab(hour='*/6'), + 'options': { + 'expires': 3600, + 'time_limit': 3600, + } + }, +} diff --git a/contributors/tasks.py b/contributors/tasks.py new file mode 100644 index 0000000..6ef7f6c --- /dev/null +++ b/contributors/tasks.py @@ -0,0 +1,19 @@ +from celery import shared_task +from celery.utils.log import get_task_logger +from django.core.management import call_command + +from contributors.management.commands.fetchdata import ORGANIZATIONS + +logger = get_task_logger(__name__) + + +@shared_task(bind=True, max_retries=3) +def sync_github_data(self, owners=None, repos=None): + try: + orgs = [org.name for org in ORGANIZATIONS] + logger.info(orgs) + call_command("fetchdata", orgs or ["hexlet"], repo=repos or []) + + except Exception as exc: + logger.error(f"Failed to sync GitHub data: {exc}") + self.retry(exc=exc, countdown=60 * (2**self.request.retries)) diff --git a/contributors/utils/misc.py b/contributors/utils/misc.py index 3541f37..e0dcc7f 100644 --- a/contributors/utils/misc.py +++ b/contributors/utils/misc.py @@ -16,7 +16,7 @@ def getenv(env_variable): """Return an environment variable or raise an exception.""" try: - return os.environ[env_variable] + return os.getenv(env_variable) except KeyError: raise ImproperlyConfigured( f"The {env_variable} setting must not be empty.", @@ -34,51 +34,51 @@ def update_or_create_record(cls, github_resp, additional_fields=None): """ cls_fields = { - 'Organization': lambda: {'name': github_resp['login']}, - 'Repository': lambda: { - 'owner_id': ( - github_resp['owner']['id'] - if github_resp['owner']['type'] == 'User' + "Organization": lambda: {"name": github_resp["login"]}, + "Repository": lambda: { + "owner_id": ( + github_resp["owner"]["id"] + if github_resp["owner"]["type"] == "User" else None ), - 'organization_id': ( - github_resp['owner']['id'] - if github_resp['owner']['type'] == 'Organization' + "organization_id": ( + github_resp["owner"]["id"] + if github_resp["owner"]["type"] == "Organization" else None ), - 'full_name': github_resp['full_name'], + "full_name": github_resp["full_name"], }, - 'Contributor': lambda: { - 'login': github_resp['login'], - 'avatar_url': github_resp['avatar_url'], + "Contributor": lambda: { + "login": github_resp["login"], + "avatar_url": github_resp["avatar_url"], }, } defaults = { - 'name': github_resp.get('name'), - 'html_url': github_resp['html_url'], + "name": github_resp.get("name"), + "html_url": github_resp["html_url"], } defaults.update(cls_fields[cls.__name__]()) defaults.update(additional_fields or {}) return cls.objects.update_or_create( - id=github_resp['id'], + id=github_resp["id"], defaults=defaults, ) def get_contributor_data(login, session=None): """Get contributor data from database or GitHub.""" - Contributor = apps.get_model('contributors.Contributor') # noqa: N806 + Contributor = apps.get_model("contributors.Contributor") # noqa: N806 try: user = Contributor.objects.get(login=login) except Contributor.DoesNotExist: return github.get_owner_data(login, session) return { - 'id': user.id, - 'name': user.name, - 'html_url': user.html_url, - 'login': user.login, - 'avatar_url': user.avatar_url, + "id": user.id, + "name": user.name, + "html_url": user.html_url, + "login": user.login, + "avatar_url": user.avatar_url, } @@ -100,13 +100,15 @@ def group_contribs_by_months(months_with_contrib_sums): """ sums_of_contribs_by_months = {} for contrib in months_with_contrib_sums: - month = sums_of_contribs_by_months.setdefault(contrib['month'], {}) - month[contrib['type']] = contrib['count'] + month = sums_of_contribs_by_months.setdefault(contrib["month"], {}) + month[contrib["type"]] = contrib["count"] return sums_of_contribs_by_months def get_rotated_sums_for_contrib( - current_month: int, sums_of_contribs_by_months, contrib_type, + current_month: int, + sums_of_contribs_by_months, + contrib_type, ): """ Return an array of 12 sums of contributions of the given type. @@ -115,16 +117,19 @@ def get_rotated_sums_for_contrib( The collection is left-shifted by the numeric value of the current month. """ months = range(1, NUM_OF_MONTHS_IN_A_YEAR + 1) - array = deque([ - sums_of_contribs_by_months.get(month, {}).get(contrib_type, 0) - for month in months - ]) + array = deque( + [ + sums_of_contribs_by_months.get(month, {}).get(contrib_type, 0) + for month in months + ] + ) array.rotate(-current_month) return list(array) def get_contrib_sums_distributed_over_months( - current_month: int, sums_of_contribs_by_months, + current_month: int, + sums_of_contribs_by_months, ): """ Return shifted monthly sums for each contribution type. @@ -137,10 +142,10 @@ def get_contrib_sums_distributed_over_months( sums_of_contribs_by_months, ) return { - 'commits': rotated_sums_by_months('cit'), - 'pull_requests': rotated_sums_by_months('pr'), - 'issues': rotated_sums_by_months('iss'), - 'comments': rotated_sums_by_months('cnt'), + "commits": rotated_sums_by_months("cit"), + "pull_requests": rotated_sums_by_months("pr"), + "issues": rotated_sums_by_months("iss"), + "comments": rotated_sums_by_months("cnt"), } @@ -159,21 +164,23 @@ def datetime_week_ago(): def split_full_name(name): """Split a full name into parts.""" if not name: - return ('', '') + return ("", "") name_parts = name.split() first_name = name_parts[0] - last_name = name_parts[-1] if len(name_parts) > 1 else '' + last_name = name_parts[-1] if len(name_parts) > 1 else "" return (first_name, last_name) def split_ordering(ordering): """Return a tuple of ordering direction and field name.""" - if ordering.startswith('-'): - return ('-', ordering[1:]) - return ('', ordering) + if ordering.startswith("-"): + return ("-", ordering[1:]) + return ("", ordering) -DIRECTION_TRANSLATIONS = MappingProxyType({ - '': 'asc', - '-': 'desc', -}) +DIRECTION_TRANSLATIONS = MappingProxyType( + { + "": "asc", + "-": "desc", + } +) diff --git a/pyproject.toml b/pyproject.toml index b59e91b..509686f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ readme = "README.md" requires-python = ">=3.11" dependencies = [ "apscheduler>=3.10.4", + "celery>=5.4.0", "crispy-bootstrap5>=2024.10", "cryptography>=43.0.3", "dj-database-url>=2.3.0", diff --git a/uv.lock b/uv.lock index ab7aa39..aeb65b3 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,18 @@ version = 1 requires-python = ">=3.11" +[[package]] +name = "amqp" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944 }, +] + [[package]] name = "apscheduler" version = "3.10.4" @@ -24,6 +36,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/39/e3/893e8757be2612e6c266d9bb58ad2e3651524b5b40cf56761e985a28b13e/asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47", size = 23828 }, ] +[[package]] +name = "billiard" +version = "4.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/58/1546c970afcd2a2428b1bfafecf2371d8951cc34b46701bea73f4280989e/billiard-4.2.1.tar.gz", hash = "sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f", size = 155031 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/da/43b15f28fe5f9e027b41c539abc5469052e9d48fd75f8ff094ba2a0ae767/billiard-4.2.1-py3-none-any.whl", hash = "sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb", size = 86766 }, +] + +[[package]] +name = "celery" +version = "5.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "billiard" }, + { name = "click" }, + { name = "click-didyoumean" }, + { name = "click-plugins" }, + { name = "click-repl" }, + { name = "kombu" }, + { name = "python-dateutil" }, + { name = "tzdata" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/9c/cf0bce2cc1c8971bf56629d8f180e4ca35612c7e79e6e432e785261a8be4/celery-5.4.0.tar.gz", hash = "sha256:504a19140e8d3029d5acad88330c541d4c3f64c789d85f94756762d8bca7e706", size = 1575692 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/c4/6a4d3772e5407622feb93dd25c86ce3c0fee746fa822a777a627d56b4f2a/celery-5.4.0-py3-none-any.whl", hash = "sha256:369631eb580cf8c51a82721ec538684994f8277637edde2dfc0dacd73ed97f64", size = 425983 }, +] + [[package]] name = "certifi" version = "2024.8.30" @@ -141,6 +182,64 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/9b/08c0432272d77b04803958a4598a51e2a4b51c06640af8b8f0f908c18bf2/charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", size = 49446 }, ] +[[package]] +name = "click" +version = "8.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "platform_system == 'Windows'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", size = 97941 }, +] + +[[package]] +name = "click-didyoumean" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631 }, +] + +[[package]] +name = "click-plugins" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5f/1d/45434f64ed749540af821fd7e42b8e4d23ac04b1eda7c26613288d6cd8a8/click-plugins-1.1.1.tar.gz", hash = "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", size = 8164 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/da/824b92d9942f4e472702488857914bdd50f73021efea15b4cad9aca8ecef/click_plugins-1.1.1-py2.py3-none-any.whl", hash = "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8", size = 7497 }, +] + +[[package]] +name = "click-repl" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289 }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + [[package]] name = "coverage" version = "7.6.7" @@ -383,6 +482,7 @@ version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "apscheduler" }, + { name = "celery" }, { name = "crispy-bootstrap5" }, { name = "cryptography" }, { name = "dj-database-url" }, @@ -418,6 +518,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "apscheduler", specifier = ">=3.10.4" }, + { name = "celery", specifier = ">=5.4.0" }, { name = "crispy-bootstrap5", specifier = ">=2024.10" }, { name = "cryptography", specifier = ">=43.0.3" }, { name = "dj-database-url", specifier = ">=2.3.0" }, @@ -468,6 +569,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, ] +[[package]] +name = "kombu" +version = "5.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "amqp" }, + { name = "tzdata" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/4d/b93fcb353d279839cc35d0012bee805ed0cf61c07587916bfc35dbfddaf1/kombu-5.4.2.tar.gz", hash = "sha256:eef572dd2fd9fc614b37580e3caeafdd5af46c1eff31e7fba89138cdb406f2cf", size = 442858 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/ec/7811a3cf9fdfee3ee88e54d08fcbc3fabe7c1b6e4059826c59d7b795651c/kombu-5.4.2-py3-none-any.whl", hash = "sha256:14212f5ccf022fc0a70453bb025a1dcc32782a588c49ea866884047d66e14763", size = 201349 }, +] + [[package]] name = "nodeenv" version = "1.9.1" @@ -511,6 +626,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/8f/496e10d51edd6671ebe0432e33ff800aa86775d2d147ce7d43389324a525/pre_commit-4.0.1-py2.py3-none-any.whl", hash = "sha256:efde913840816312445dc98787724647c65473daefe420785f885e8ed9a06878", size = 218713 }, ] +[[package]] +name = "prompt-toolkit" +version = "3.0.48" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/4f/feb5e137aff82f7c7f3248267b97451da3644f6cdc218edfe549fb354127/prompt_toolkit-3.0.48.tar.gz", hash = "sha256:d6623ab0477a80df74e646bdbc93621143f5caf104206aa29294d53de1a03d90", size = 424684 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl", hash = "sha256:f49a827f90062e411f1ce1f854f2aedb3c23353244f8108b89283587397ac10e", size = 386595 }, +] + [[package]] name = "psycopg2-binary" version = "2.9.10" @@ -798,6 +925,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", size = 126338 }, ] +[[package]] +name = "vine" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636 }, +] + [[package]] name = "virtualenv" version = "20.27.1" @@ -812,6 +948,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/92/78324ff89391e00c8f4cf6b8526c41c6ef36b4ea2d2c132250b1a6fc2b8d/virtualenv-20.27.1-py3-none-any.whl", hash = "sha256:f11f1b8a29525562925f745563bfd48b189450f61fb34c4f9cc79dd5aa32a1f4", size = 3117838 }, ] +[[package]] +name = "wcwidth" +version = "0.2.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5", size = 101301 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166 }, +] + [[package]] name = "whitenoise" version = "6.8.2"