diff --git a/tests/unit/packaging/test_init.py b/tests/unit/packaging/test_init.py index 059a6ff069d8..cf9a48bf1941 100644 --- a/tests/unit/packaging/test_init.py +++ b/tests/unit/packaging/test_init.py @@ -22,15 +22,13 @@ from warehouse.packaging.models import File, Project, Release, Role from warehouse.packaging.tasks import ( # sync_bigquery_release_files, compute_2fa_mandate, - compute_trending, update_description_html, ) -@pytest.mark.parametrize("with_trending", [True, False]) @pytest.mark.parametrize("with_bq_sync", [True, False]) @pytest.mark.parametrize("with_2fa_mandate", [True, False]) -def test_includeme(monkeypatch, with_trending, with_bq_sync, with_2fa_mandate): +def test_includeme(monkeypatch, with_bq_sync, with_2fa_mandate): storage_class = pretend.stub( create_service=pretend.call_recorder(lambda *a, **kw: pretend.stub()) ) @@ -40,8 +38,6 @@ def key_factory(keystring, iterate_on=None): monkeypatch.setattr(packaging, "key_factory", key_factory) settings = dict() - if with_trending: - settings["warehouse.trending_table"] = "foobar" if with_bq_sync: settings["warehouse.release_files_table"] = "fizzbuzz" if with_2fa_mandate: @@ -128,12 +124,6 @@ def key_factory(keystring, iterate_on=None): # ) pass - if with_trending: - assert ( - pretend.call(crontab(minute=0, hour=3), compute_trending) - in config.add_periodic_task.calls - ) - if with_2fa_mandate: assert ( pretend.call(crontab(minute=0, hour=3), compute_2fa_mandate) diff --git a/tests/unit/packaging/test_search.py b/tests/unit/packaging/test_search.py index bf1919751040..26ea0a21e433 100644 --- a/tests/unit/packaging/test_search.py +++ b/tests/unit/packaging/test_search.py @@ -35,7 +35,6 @@ def test_build_search(): platform="any platform", created=datetime.datetime(1956, 1, 31), classifiers=["Alpha", "Beta"], - zscore=None, ) obj = Project.from_db(release) @@ -55,4 +54,3 @@ def test_build_search(): assert obj["platform"] == "any platform" assert obj["created"] == datetime.datetime(1956, 1, 31) assert obj["classifiers"] == ["Alpha", "Beta"] - assert obj["zscore"] is None diff --git a/tests/unit/packaging/test_tasks.py b/tests/unit/packaging/test_tasks.py index 83e053ab0c2e..b89f7bab5083 100644 --- a/tests/unit/packaging/test_tasks.py +++ b/tests/unit/packaging/test_tasks.py @@ -15,18 +15,16 @@ import pretend import pytest -from google.cloud.bigquery import Row, SchemaField +from google.cloud.bigquery import SchemaField from wtforms import Field, Form, StringField import warehouse.packaging.tasks from warehouse.accounts.models import WebAuthn -from warehouse.cache.origin import IOriginCache -from warehouse.packaging.models import Description, Project +from warehouse.packaging.models import Description from warehouse.packaging.tasks import ( compute_2fa_mandate, compute_2fa_metrics, - compute_trending, sync_bigquery_release_files, update_bigquery_release_files, update_description_html, @@ -45,94 +43,6 @@ ) -class TestComputeTrending: - @pytest.mark.parametrize("with_purges", [True, False]) - def test_computes_trending(self, db_request, with_purges): - projects = [ - ProjectFactory.create(zscore=1 if not i else None) for i in range(3) - ] - - results = iter( - [ - Row((projects[1].normalized_name, 2), {"project": 0, "zscore": 1}), - Row((projects[2].normalized_name, -1), {"project": 0, "zscore": 1}), - ] - ) - query = pretend.stub(result=pretend.call_recorder(lambda *a, **kw: results)) - bigquery = pretend.stub(query=pretend.call_recorder(lambda q: query)) - - cacher = pretend.stub(purge=pretend.call_recorder(lambda keys: None)) - - def find_service(iface=None, name=None): - if iface is None and name == "gcloud.bigquery": - return bigquery - - if with_purges and issubclass(iface, IOriginCache): - return cacher - - raise LookupError - - db_request.find_service = find_service - db_request.registry.settings = { - "warehouse.trending_table": "example.pypi.downloads*" - } - - compute_trending(db_request) - - assert bigquery.query.calls == [ - pretend.call( - """ SELECT project, - IF( - STDDEV(downloads) > 0, - (todays_downloads - AVG(downloads))/STDDEV(downloads), - NULL - ) as zscore - FROM ( - SELECT project, - date, - downloads, - FIRST_VALUE(downloads) OVER ( - PARTITION BY project - ORDER BY DATE DESC - ROWS BETWEEN UNBOUNDED PRECEDING - AND UNBOUNDED FOLLOWING - ) as todays_downloads - FROM ( - SELECT file.project as project, - DATE(timestamp) AS date, - COUNT(*) as downloads - FROM `example.pypi.downloads*` - WHERE _TABLE_SUFFIX BETWEEN - FORMAT_DATE( - "%Y%m%d", - DATE_ADD(CURRENT_DATE(), INTERVAL -31 day)) - AND - FORMAT_DATE( - "%Y%m%d", - DATE_ADD(CURRENT_DATE(), INTERVAL -1 day)) - GROUP BY file.project, date - ) - ) - GROUP BY project, todays_downloads - HAVING SUM(downloads) >= 5000 - ORDER BY zscore DESC - """ - ) - ] - assert query.result.calls == [pretend.call()] - assert cacher.purge.calls == ( - [pretend.call(["trending"])] if with_purges else [] - ) - - results = dict(db_request.db.query(Project.name, Project.zscore).all()) - - assert results == { - projects[0].name: None, - projects[1].name: 2, - projects[2].name: -1, - } - - def test_update_description_html(monkeypatch, db_request): current_version = "24.0" previous_version = "23.0" diff --git a/tests/unit/test_search.py b/tests/unit/test_search.py index 7884dc0565c7..b25b188bc6b4 100644 --- a/tests/unit/test_search.py +++ b/tests/unit/test_search.py @@ -142,9 +142,7 @@ def test_mixed_quoted_query(self): }, } - @pytest.mark.parametrize( - "order,field", [("created", "created"), ("-zscore", "zscore")] - ) + @pytest.mark.parametrize("order,field", [("created", "created")]) def test_sort_order(self, order, field): es = Search() terms = "foo bar" diff --git a/tests/unit/test_views.py b/tests/unit/test_views.py index 69fc00054928..3489a1581ef9 100644 --- a/tests/unit/test_views.py +++ b/tests/unit/test_views.py @@ -327,9 +327,6 @@ def test_index(self, db_request): UserFactory.create() assert index(db_request) == { - # assert that ordering is correct - "latest_releases": [release2, release1], - "trending_projects": [release2], "num_projects": 1, "num_users": 3, "num_releases": 2, diff --git a/warehouse/config.py b/warehouse/config.py index 8df50efb8447..64546a7d1e7c 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -176,7 +176,6 @@ def configure(settings=None): "GITHUB_TOKEN_SCANNING_META_API_URL", default="https://api.github.com/meta/public_keys/token_scanning", ) - maybe_set(settings, "warehouse.trending_table", "WAREHOUSE_TRENDING_TABLE") maybe_set(settings, "warehouse.downloads_table", "WAREHOUSE_DOWNLOADS_TABLE") maybe_set(settings, "celery.broker_url", "BROKER_URL") maybe_set(settings, "celery.result_url", "REDIS_URL") diff --git a/warehouse/locale/messages.pot b/warehouse/locale/messages.pot index 2b06acf0880d..f7c8536ad8bd 100644 --- a/warehouse/locale/messages.pot +++ b/warehouse/locale/messages.pot @@ -1,10 +1,10 @@ -#: warehouse/views.py:141 +#: warehouse/views.py:140 msgid "" "Two-factor authentication must be enabled on your account to perform this" " action." msgstr "" -#: warehouse/views.py:304 +#: warehouse/views.py:265 msgid "Locale updated" msgstr "" @@ -520,7 +520,7 @@ msgstr "" #: warehouse/templates/includes/accounts/profile-callout.html:18 #: warehouse/templates/includes/hash-modal.html:23 #: warehouse/templates/includes/packaging/project-data.html:66 -#: warehouse/templates/index.html:112 warehouse/templates/index.html:116 +#: warehouse/templates/index.html:100 warehouse/templates/index.html:104 #: warehouse/templates/manage/account.html:228 #: warehouse/templates/manage/account.html:234 #: warehouse/templates/manage/account/totp-provision.html:32 @@ -737,7 +737,7 @@ msgid "" "Foundation." msgstr "" -#: warehouse/templates/base.html:93 warehouse/templates/index.html:109 +#: warehouse/templates/base.html:93 warehouse/templates/index.html:97 msgid "" "The Python Package Index (PyPI) is a repository of software for the " "Python programming language." @@ -796,19 +796,19 @@ msgstr "" #: warehouse/templates/base.html:227 warehouse/templates/base.html:248 #: warehouse/templates/error-base-with-search.html:20 -#: warehouse/templates/index.html:55 +#: warehouse/templates/index.html:43 msgid "Search PyPI" msgstr "" #: warehouse/templates/base.html:228 warehouse/templates/base.html:249 #: warehouse/templates/error-base-with-search.html:21 -#: warehouse/templates/index.html:58 +#: warehouse/templates/index.html:46 msgid "Search projects" msgstr "" #: warehouse/templates/base.html:232 warehouse/templates/base.html:253 #: warehouse/templates/error-base-with-search.html:24 -#: warehouse/templates/index.html:62 +#: warehouse/templates/index.html:50 msgid "Search" msgstr "" @@ -971,85 +971,69 @@ msgstr "" msgid "The Python Package Index" msgstr "" -#: warehouse/templates/index.html:43 +#: warehouse/templates/index.html:31 msgid "Test Python package publishing with the Test Python Package Index" msgstr "" -#: warehouse/templates/index.html:45 +#: warehouse/templates/index.html:33 msgid "Develop the codebase behind PyPI with the Dev Python Package Index" msgstr "" -#: warehouse/templates/index.html:47 +#: warehouse/templates/index.html:35 msgid "Find, install and publish Python packages with the Python Package Index" msgstr "" -#: warehouse/templates/index.html:66 +#: warehouse/templates/index.html:54 #, python-format msgid "Or browse projects" msgstr "" -#: warehouse/templates/index.html:73 +#: warehouse/templates/index.html:61 #, python-format msgid "%(num_projects_formatted)s project" msgid_plural "%(num_projects_formatted)s projects" msgstr[0] "" msgstr[1] "" -#: warehouse/templates/index.html:80 +#: warehouse/templates/index.html:68 #, python-format msgid "%(num_releases_formatted)s release" msgid_plural "%(num_releases_formatted)s releases" msgstr[0] "" msgstr[1] "" -#: warehouse/templates/index.html:87 +#: warehouse/templates/index.html:75 #, python-format msgid "%(num_files_formatted)s file" msgid_plural "%(num_files_formatted)s files" msgstr[0] "" msgstr[1] "" -#: warehouse/templates/index.html:94 +#: warehouse/templates/index.html:82 #, python-format msgid "%(num_users_formatted)s user" msgid_plural "%(num_users_formatted)s users" msgstr[0] "" msgstr[1] "" -#: warehouse/templates/index.html:111 +#: warehouse/templates/index.html:99 msgid "" "PyPI helps you find and install software developed and shared by the " "Python community." msgstr "" -#: warehouse/templates/index.html:112 +#: warehouse/templates/index.html:100 msgid "Learn about installing packages" msgstr "" -#: warehouse/templates/index.html:115 +#: warehouse/templates/index.html:103 msgid "Package authors use PyPI to distribute their software." msgstr "" -#: warehouse/templates/index.html:116 +#: warehouse/templates/index.html:104 msgid "Learn how to package your Python code for PyPI" msgstr "" -#: warehouse/templates/index.html:126 -msgid "Trending projects" -msgstr "" - -#: warehouse/templates/index.html:127 -msgid "Trending projects as downloaded by the community" -msgstr "" - -#: warehouse/templates/index.html:137 -msgid "New releases" -msgstr "" - -#: warehouse/templates/index.html:138 -msgid "Hot off the press: the newest project releases" -msgstr "" - #: warehouse/templates/accounts/login.html:17 #: warehouse/templates/accounts/recovery-code.html:17 #: warehouse/templates/accounts/register.html:17 @@ -2347,7 +2331,7 @@ msgstr "" #: warehouse/templates/manage/project/release.html:137 #: warehouse/templates/manage/project/releases.html:178 #: warehouse/templates/manage/project/settings.html:111 -#: warehouse/templates/search/results.html:199 +#: warehouse/templates/search/results.html:198 msgid "Close" msgstr "" @@ -7789,7 +7773,7 @@ msgstr "" #: warehouse/templates/search/results.html:18 #: warehouse/templates/search/results.html:114 -#: warehouse/templates/search/results.html:212 +#: warehouse/templates/search/results.html:211 msgid "Search results" msgstr "" @@ -7861,24 +7845,20 @@ msgstr "" msgid "Date last updated" msgstr "" -#: warehouse/templates/search/results.html:181 -msgid "Trending" -msgstr "" - -#: warehouse/templates/search/results.html:194 +#: warehouse/templates/search/results.html:193 msgid "Filter" msgstr "" -#: warehouse/templates/search/results.html:205 +#: warehouse/templates/search/results.html:204 msgid "Add filter" msgstr "" -#: warehouse/templates/search/results.html:221 +#: warehouse/templates/search/results.html:220 #, python-format msgid "There were no results for '%(term)s'" msgstr "" -#: warehouse/templates/search/results.html:223 +#: warehouse/templates/search/results.html:222 #, python-format msgid "There were no results for '%(filters)s' filter" msgid_plural "There were no results for '%(filters)s' filters" diff --git a/warehouse/packaging/__init__.py b/warehouse/packaging/__init__.py index 2a7092600576..88d68f3546d5 100644 --- a/warehouse/packaging/__init__.py +++ b/warehouse/packaging/__init__.py @@ -22,7 +22,6 @@ from warehouse.packaging.tasks import ( compute_2fa_mandate, compute_2fa_metrics, - compute_trending, update_description_html, ) @@ -110,11 +109,6 @@ def includeme(config): # Add a periodic task to generate 2FA metrics config.add_periodic_task(crontab(minute="*/5"), compute_2fa_metrics) - # Add a periodic task to compute trending once a day, assuming we have - # been configured to be able to access BigQuery. - if config.get_settings().get("warehouse.trending_table"): - config.add_periodic_task(crontab(minute=0, hour=3), compute_trending) - # TODO: restore this # if config.get_settings().get("warehouse.release_files_table"): # config.add_periodic_task(crontab(minute=0), sync_bigquery_release_files) diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 801726ba0111..5405c24e4437 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -29,7 +29,6 @@ DateTime, Enum, FetchedValue, - Float, ForeignKey, Index, Integer, @@ -196,7 +195,6 @@ class Project(SitemapMixin, TwoFactorRequireable, HasEvents, db.Model): upload_limit = Column(Integer, nullable=True) total_size_limit = Column(BigInteger, nullable=True) last_serial = Column(Integer, nullable=False, server_default=sql.text("0")) - zscore = Column(Float, nullable=True) total_size = Column(BigInteger, server_default=sql.text("0")) organization = orm.relationship( diff --git a/warehouse/packaging/search.py b/warehouse/packaging/search.py index 5d950e8557cf..8208228cbb2b 100644 --- a/warehouse/packaging/search.py +++ b/warehouse/packaging/search.py @@ -12,7 +12,7 @@ import packaging.version -from elasticsearch_dsl import Date, Document, Float, Keyword, Text, analyzer +from elasticsearch_dsl import Date, Document, Keyword, Text, analyzer from warehouse.search.utils import doc_type @@ -49,7 +49,6 @@ class Project(Document): platform = Keyword() created = Date() classifiers = Keyword(multi=True) - zscore = Float() @classmethod def from_db(cls, release): @@ -72,7 +71,6 @@ def from_db(cls, release): obj["platform"] = release.platform obj["created"] = release.created obj["classifiers"] = release.classifiers - obj["zscore"] = release.zscore return obj diff --git a/warehouse/packaging/tasks.py b/warehouse/packaging/tasks.py index f7229c0e1d68..64473243c3e4 100644 --- a/warehouse/packaging/tasks.py +++ b/warehouse/packaging/tasks.py @@ -21,7 +21,6 @@ from warehouse import tasks from warehouse.accounts.models import User, WebAuthn -from warehouse.cache.origin import IOriginCache from warehouse.email import send_two_factor_mandate_email from warehouse.metrics import IMetricsService from warehouse.packaging.models import Description, File, Project, Release, Role @@ -191,87 +190,6 @@ def compute_2fa_metrics(request): ) -@tasks.task(ignore_result=True, acks_late=True) -def compute_trending(request): - bq = request.find_service(name="gcloud.bigquery") - query = bq.query( - """ SELECT project, - IF( - STDDEV(downloads) > 0, - (todays_downloads - AVG(downloads))/STDDEV(downloads), - NULL - ) as zscore - FROM ( - SELECT project, - date, - downloads, - FIRST_VALUE(downloads) OVER ( - PARTITION BY project - ORDER BY DATE DESC - ROWS BETWEEN UNBOUNDED PRECEDING - AND UNBOUNDED FOLLOWING - ) as todays_downloads - FROM ( - SELECT file.project as project, - DATE(timestamp) AS date, - COUNT(*) as downloads - FROM `{table}` - WHERE _TABLE_SUFFIX BETWEEN - FORMAT_DATE( - "%Y%m%d", - DATE_ADD(CURRENT_DATE(), INTERVAL -31 day)) - AND - FORMAT_DATE( - "%Y%m%d", - DATE_ADD(CURRENT_DATE(), INTERVAL -1 day)) - GROUP BY file.project, date - ) - ) - GROUP BY project, todays_downloads - HAVING SUM(downloads) >= 5000 - ORDER BY zscore DESC - """.format( - table=request.registry.settings["warehouse.trending_table"] - ) - ) - - zscores = {} - for row in query.result(): - row = dict(row) - zscores[row["project"]] = row["zscore"] - - # We're going to "reset" all of our zscores to a steady state where they - # are all equal to ``None``. The next query will then set any that have a - # value back to the expected value. - ( - request.db.query(Project) - .filter(Project.zscore != None) # noqa - .update({Project.zscore: None}) - ) - - # We need to convert the normalized name that we get out of BigQuery and - # turn it into the primary key of the Project object and construct a list - # of primary key: new zscore, including a default of None if the item isn't - # in the result set. - query = request.db.query(Project.id, Project.normalized_name).all() - to_update = [ - {"id": id, "zscore": zscores[normalized_name]} - for id, normalized_name in query - if normalized_name in zscores - ] - - # Reflect out updated ZScores into the database. - request.db.bulk_update_mappings(Project, to_update) - - # Trigger a purge of the trending surrogate key. - try: - cacher = request.find_service(IOriginCache) - except LookupError: - pass - else: - cacher.purge(["trending"]) - - @tasks.task(ignore_result=True, acks_late=True) def update_description_html(request): renderer_version = readme.renderer_version() diff --git a/warehouse/search/tasks.py b/warehouse/search/tasks.py index cc5ae642ab88..eab29f1181ea 100644 --- a/warehouse/search/tasks.py +++ b/warehouse/search/tasks.py @@ -93,7 +93,6 @@ def _project_docs(db, project_name=None): classifiers, Project.normalized_name, Project.name, - Project.zscore, ) .select_from(releases_list) .join(Release, Release.id == releases_list.c.id) diff --git a/warehouse/templates/index.html b/warehouse/templates/index.html index 5b019f9e60c6..4ed7e24d749f 100644 --- a/warehouse/templates/index.html +++ b/warehouse/templates/index.html @@ -15,18 +15,6 @@ {% block title_base %}{{ request.registry.settings['site.name'] }} ยท {% trans %}The Python Package Index{% endtrans %}{% endblock %} -{% macro project_snippet(release) -%} -
  • - -

    - {{ release.project.name }} - {{ release.version }} -

    -

    {{ release.summary }}

    -     -
    • -{%- endmacro %} - {% block brand %}