From b91332d9929216d1939888e8a37b448f7dbeaa0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Ja=C5=A1ek?= Date: Wed, 31 Jul 2024 09:46:10 +0200 Subject: [PATCH] bump superdesk-core and fix eve events issues (#1009) * bump superdesk-core - make getattr only handle `on_` callbacks as events - fix issues that were hidden due to it * use superdesk-core v2.7.3 --- .github/workflows/tests.yml | 6 +- dev-requirements.in | 11 + dev-requirements.txt | 537 +++++++++++++++++++++++++++++++- e2e/server/Dockerfile | 7 +- newsroom/factory/app.py | 10 + newsroom/monitoring/__init__.py | 1 - newsroom/news_api/__init__.py | 2 +- newsroom/news_api/factory.py | 5 +- newsroom/upload.py | 1 + newsroom/web/factory.py | 1 - requirements.txt | 6 +- 11 files changed, 562 insertions(+), 25 deletions(-) create mode 100644 dev-requirements.in diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4adaf02ce..fc69fe9fd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,10 +27,8 @@ jobs: sudo apt-get update sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl - - name: pip install - run: | - python -m pip install --upgrade pip wheel setuptools - pip install -r dev-requirements.txt + - run: python -m pip install --upgrade pip wheel setuptools + - run: python -m pip install -r dev-requirements.txt - name: pytest run: pytest --ignore=tests/aap/ --disable-pytest-warnings --cov=newsroom diff --git a/dev-requirements.in b/dev-requirements.in new file mode 100644 index 000000000..b23a871ac --- /dev/null +++ b/dev-requirements.in @@ -0,0 +1,11 @@ +-r requirements.txt +-r mypy-requirements.txt +-r black-requirements.txt + +flake8 +pytest +pytest-cov +pytest-mock +responses>=0.10.6,<0.26 +httmock +wooper diff --git a/dev-requirements.txt b/dev-requirements.txt index b23a871ac..377b41454 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,11 +1,528 @@ --r requirements.txt --r mypy-requirements.txt --r black-requirements.txt +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile dev-requirements.in +# +amqp==5.2.0 + # via kombu +arabic-reshaper==3.0.0 + # via xhtml2pdf +arrow==0.13.0 + # via + # eve-elastic + # superdesk-core +asn1crypto==1.5.1 + # via + # oscrypto + # pyhanko + # pyhanko-certvalidator +async-timeout==4.0.3 + # via redis +authlib==0.14.3 + # via superdesk-core +babel==2.15.0 + # via flask-babel +bcrypt==4.1.3 + # via superdesk-core +behave==1.2.6 + # via wooper +billiard==3.6.4.0 + # via celery +black==23.12.1 + # via -r black-requirements.txt +blinker==1.7.0 + # via + # elastic-apm + # flask-mail + # raven + # sentry-sdk + # superdesk-core +boto3==1.34.150 + # via superdesk-core +botocore==1.34.150 + # via + # boto3 + # s3transfer +cachelib==0.9.0 + # via flask-caching +cachetools==5.4.0 + # via + # flask-oidc-ex + # google-auth +celery[redis]==5.2.7 + # via superdesk-core +cerberus==1.3.5 + # via + # eve + # superdesk-core +certifi==2024.7.4 + # via + # elastic-apm + # elasticsearch + # requests + # sentry-sdk +cffi==1.16.0 + # via cryptography +chardet==5.2.0 + # via superdesk-core +charset-normalizer==3.3.2 + # via requests +ciso8601==2.3.1 + # via eve-elastic +click==8.1.7 + # via + # -r requirements.txt + # black + # celery + # click-didyoumean + # click-plugins + # click-repl + # flask + # pyhanko + # superdesk-core +click-didyoumean==0.3.1 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.3.0 + # via celery +coverage[toml]==7.6.0 + # via pytest-cov +croniter==2.0.7 + # via superdesk-core +cryptography==43.0.0 + # via + # authlib + # jwcrypto + # pyhanko + # pyhanko-certvalidator +cssselect2==0.7.0 + # via svglib +deepdiff==7.0.1 + # via superdesk-planning +deprecated==1.2.14 + # via limits +dnspython==2.6.1 + # via email-validator +draftjs-exporter[lxml]==2.1.7 + # via superdesk-core +ecs-logging==2.2.0 + # via elastic-apm +elastic-apm[flask]==6.22.3 + # via superdesk-core +elasticsearch==7.13.4 + # via + # eve-elastic + # superdesk-core +email-validator==2.2.0 + # via wtforms +eve==2.1.0 + # via superdesk-core +eve-elastic==7.4.1 + # via + # -r requirements.txt + # superdesk-core +events==0.3 + # via eve +exceptiongroup==1.2.2 + # via pytest +feedparser==6.0.11 + # via superdesk-core +flake8==7.1.0 + # via -r dev-requirements.in +flask==1.1.2 + # via + # -r requirements.txt + # eve + # flask-babel + # flask-caching + # flask-limiter + # flask-mail + # flask-oidc-ex + # flask-pymongo + # flask-script + # flask-webpack + # flask-wtf + # raven + # sentry-sdk + # superdesk-core +flask-babel==2.0.0 + # via superdesk-core +flask-caching==2.3.0 + # via -r requirements.txt +flask-limiter==0.9.5.1 + # via -r requirements.txt +flask-mail==0.9.1 + # via superdesk-core +flask-oidc-ex==0.5.5 + # via superdesk-core +flask-pymongo==2.3.0 + # via -r requirements.txt +flask-script==2.0.6 + # via superdesk-core +flask-webpack==0.1.0 + # via -r requirements.txt +flask-wtf==1.2.1 + # via -r requirements.txt +future==1.0.0 + # via python-twitter +google-auth==2.29.0 + # via -r requirements.txt +gunicorn==22.0.0 + # via -r requirements.txt +hachoir==3.0a3 + # via superdesk-core +hermescache==0.10.0 + # via superdesk-core +honcho==1.1.0 + # via -r requirements.txt +html5lib==1.1 + # via xhtml2pdf +httmock==1.4.0 + # via -r dev-requirements.in +httplib2==0.22.0 + # via oauth2client +icalendar==4.0.9 + # via superdesk-planning +idna==3.7 + # via + # email-validator + # requests +importlib-resources==6.4.0 + # via limits +iniconfig==2.0.0 + # via pytest +isodate==0.6.1 + # via python3-saml +itsdangerous==1.1.0 + # via + # flask + # flask-oidc-ex + # flask-wtf + # superdesk-core +jinja2==2.11.3 + # via + # flask + # flask-babel + # superdesk-core +jmespath==1.0.1 + # via + # boto3 + # botocore +jwcrypto==1.5.6 + # via + # flask-oidc-ex + # python-jwt +kombu==5.2.4 + # via + # celery + # superdesk-core +ldap3==2.5.2 + # via superdesk-core +limits==3.13.0 + # via flask-limiter +lxml==4.6.5 + # via + # draftjs-exporter + # python3-saml + # superdesk-core + # svglib + # xmlsec +markupsafe==2.0.1 + # via + # -r requirements.txt + # jinja2 + # sentry-sdk + # superdesk-core + # wtforms +mccabe==0.7.0 + # via flake8 +mongolock @ git+https://github.com/superdesk/mongolock.git@v1 + # via superdesk-core +mypy==1.11.0 + # via -r mypy-requirements.txt +mypy-extensions==1.0.0 + # via + # black + # mypy +oauth2client==4.1.3 + # via flask-oidc-ex +oauthlib==3.2.2 + # via requests-oauthlib +ordered-set==4.1.0 + # via deepdiff +oscrypto==1.3.0 + # via pyhanko-certvalidator +packaging==24.1 + # via + # black + # gunicorn + # limits + # pytest +parse==1.20.2 + # via + # behave + # parse-type +parse-type==0.6.2 + # via behave +pathspec==0.12.1 + # via black +pillow==9.2.0 + # via + # reportlab + # superdesk-core + # xhtml2pdf +platformdirs==4.2.2 + # via black +pluggy==1.5.0 + # via pytest +prompt-toolkit==3.0.47 + # via click-repl +pyasn1==0.6.0 + # via + # ldap3 + # oauth2client + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via + # google-auth + # oauth2client +pycodestyle==2.12.0 + # via flake8 +pycparser==2.22 + # via cffi +pyflakes==3.2.0 + # via flake8 +pyhanko==0.18.0 + # via xhtml2pdf +pyhanko-certvalidator==0.22.0 + # via + # pyhanko + # xhtml2pdf +pyjwt==2.4.0 + # via superdesk-core +pymemcache==4.0.0 + # via superdesk-core +pymongo==3.11.4 + # via + # eve + # flask-pymongo + # mongolock + # superdesk-core +pyparsing==3.1.2 + # via + # httplib2 + # pyrtf3 +pypdf==4.3.1 + # via xhtml2pdf +pypng==0.20220715.0 + # via qrcode +pyrtf3==0.47.5 + # via -r requirements.txt +pytest==8.3.2 + # via + # -r dev-requirements.in + # pytest-cov + # pytest-mock +pytest-cov==5.0.0 + # via -r dev-requirements.in +pytest-mock==3.14.0 + # via -r dev-requirements.in +python-bidi==0.5.1 + # via xhtml2pdf +python-dateutil==2.9.0.post0 + # via + # arrow + # botocore + # croniter + # icalendar + # superdesk-core +python-jwt==4.1.0 + # via flask-oidc-ex +python-magic==0.4.27 + # via superdesk-core +python-twitter==3.5 + # via superdesk-core +python3-saml==1.16.0 + # via -r requirements.txt +pytz==2024.1 + # via + # celery + # croniter + # eve-elastic + # flask-babel + # icalendar + # pyhanko + # superdesk-core + # tzlocal +pyyaml==6.0.1 + # via + # pyhanko + # responses + # superdesk-core +qrcode==7.4.2 + # via pyhanko +raven[flask]==6.10.0 + # via superdesk-core +redis==5.0.7 + # via + # celery + # superdesk-core +regex==2020.7.14 + # via superdesk-core +reportlab==3.6.13 + # via + # -r requirements.txt + # superdesk-core + # svglib + # xhtml2pdf +requests==2.32.3 + # via + # httmock + # pyhanko + # pyhanko-certvalidator + # python-twitter + # requests-oauthlib + # responses + # superdesk-core + # wooper +requests-oauthlib==2.0.0 + # via python-twitter +responses==0.25.3 + # via -r dev-requirements.in +rsa==4.9 + # via + # google-auth + # oauth2client +s3transfer==0.10.2 + # via boto3 +sentry-sdk[flask]==2.11.0 + # via -r requirements.txt +sgmllib3k==1.0.0 + # via feedparser +simplejson==3.19.2 + # via eve +six==1.16.0 + # via + # behave + # flask-limiter + # flask-oidc-ex + # html5lib + # isodate + # oauth2client + # parse-type + # python-dateutil +superdesk-core @ git+https://github.com/superdesk/superdesk-core.git@v2.7.3 + # via -r requirements.txt +superdesk-planning @ git+https://github.com/superdesk/superdesk-planning.git@v2.7.2 + # via -r requirements.txt +svglib==1.5.1 + # via xhtml2pdf +tinycss2==1.3.0 + # via + # cssselect2 + # svglib +tomli==2.0.1 + # via + # black + # coverage + # mypy + # pytest +types-click==7.1.8 + # via types-flask +types-flask==1.1.6 + # via -r mypy-requirements.txt +types-itsdangerous==1.1.6 + # via -r mypy-requirements.txt +types-jinja2==2.11.9 + # via + # -r mypy-requirements.txt + # types-flask +types-markupsafe==1.1.10 + # via types-jinja2 +types-protobuf==5.27.0.20240626 + # via -r mypy-requirements.txt +types-python-dateutil==2.9.0.20240316 + # via -r mypy-requirements.txt +types-pytz==2024.1.0.20240417 + # via + # -r mypy-requirements.txt + # types-tzlocal +types-requests==2.31.0.6 + # via -r mypy-requirements.txt +types-tzlocal==5.1.0.1 + # via -r mypy-requirements.txt +types-urllib3==1.26.25.14 + # via types-requests +types-werkzeug==1.0.9 + # via + # -r mypy-requirements.txt + # types-flask +typing-extensions==4.12.2 + # via + # -r mypy-requirements.txt + # black + # jwcrypto + # limits + # mypy + # pypdf + # qrcode + # superdesk-core +tzlocal==2.1 + # via + # pyhanko + # superdesk-core +unidecode==1.3.8 + # via superdesk-core +uritools==4.0.3 + # via pyhanko-certvalidator +urllib3==1.26.19 + # via + # botocore + # elastic-apm + # elasticsearch + # requests + # responses + # sentry-sdk + # superdesk-core +vine==5.1.0 + # via + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via prompt-toolkit +webencodings==0.5.1 + # via + # cssselect2 + # html5lib + # tinycss2 +websockets==10.3 + # via superdesk-core +werkzeug==1.0.1 + # via + # flask + # superdesk-core +wooper==0.4.4 + # via -r dev-requirements.in +wrapt==1.16.0 + # via + # deprecated + # elastic-apm +wtforms[email]==3.1.2 + # via + # -r requirements.txt + # flask-wtf + # wtforms +xhtml2pdf==0.2.11 + # via -r requirements.txt +xmlsec==1.3.13 + # via + # -r requirements.txt + # python3-saml + # superdesk-core -flake8 -pytest -pytest-cov -pytest-mock -responses>=0.10.6,<0.26 -httmock -wooper +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/e2e/server/Dockerfile b/e2e/server/Dockerfile index f9a39ad52..9e8149783 100644 --- a/e2e/server/Dockerfile +++ b/e2e/server/Dockerfile @@ -27,12 +27,11 @@ RUN python3 -m pip install -U pip wheel setuptools # install requirements WORKDIR /opt/newsroom/ -COPY requirements.txt . -RUN python3 -m pip install -Ur requirements.txt +COPY requirements.txt setup.py /opt/newsroom/ +RUN python3 -m pip install -Ue . -# install newsroom app +# copy app COPY . . -RUN python3 -m pip install -e . # setup entrypoint WORKDIR /opt/newsroom/e2e/server/ diff --git a/newsroom/factory/app.py b/newsroom/factory/app.py index 977a591b2..273e665f8 100644 --- a/newsroom/factory/app.py +++ b/newsroom/factory/app.py @@ -17,6 +17,7 @@ from flask_mail import Mail from flask_caching import Cache +from eve.io.mongo import ensure_mongo_indexes from superdesk.storage import AmazonMediaStorage, SuperdeskGridFSMediaStorage from superdesk.datalayer import SuperdeskDataLayer from superdesk.json_utils import SuperdeskJSONEncoder @@ -44,6 +45,11 @@ class BaseNewsroomApp(eve.Eve): AUTH_SERVICE = SessionAuth INSTANCE_CONFIG = None + def __getattr__(self, name): + if name.startswith("on_"): + return super(BaseNewsroomApp, self).__getattr__(name) + raise AttributeError("type object '%s' has no attribute '%s'" % (self.__class__.__name__, name)) + def __init__(self, import_name=__package__, config=None, testing=False, **kwargs): """Override __init__ to do Newsroom specific config and still be able to create an instance using ``app = Newsroom()`` @@ -269,3 +275,7 @@ def _get_apm_environment(self): if "localhost" in self.config["CLIENT_URL"] or self.debug: return "testing" return "production" + + def init_indexes(self): + for resource in self.config["DOMAIN"]: + ensure_mongo_indexes(self, resource) diff --git a/newsroom/monitoring/__init__.py b/newsroom/monitoring/__init__.py index e2902b4de..d8c3cf50a 100644 --- a/newsroom/monitoring/__init__.py +++ b/newsroom/monitoring/__init__.py @@ -42,4 +42,3 @@ def init_app(app): superdesk.register_resource("monitoring_search", MonitoringSearchResource, MonitoringSearchService, _app=app) app.add_template_global(get_keywords_in_text, "get_keywords_in_text") - app.add_template_global(app.theme_folder, "monitoring_image_path") diff --git a/newsroom/news_api/__init__.py b/newsroom/news_api/__init__.py index 7966fe68c..90cf22dc5 100644 --- a/newsroom/news_api/__init__.py +++ b/newsroom/news_api/__init__.py @@ -5,7 +5,7 @@ def init_app(app): if not app.config.get("NEWS_API_ENABLED"): return - if getattr(app, "section"): + if hasattr(app, "section"): app.section("news_api", "News API", "api") app.general_setting( diff --git a/newsroom/news_api/factory.py b/newsroom/news_api/factory.py index 53bbf9e2c..dbd36009b 100644 --- a/newsroom/news_api/factory.py +++ b/newsroom/news_api/factory.py @@ -33,7 +33,7 @@ class NewsroomNewsAPI(BaseNewsroomApp): INSTANCE_CONFIG = "settings_newsapi.py" def __init__(self, import_name=__package__, config=None, **kwargs): - if not getattr(self, "settings"): + if not hasattr(self, "settings"): self.settings = flask.Config(".") if config and config.get("BEHAVE"): @@ -133,6 +133,9 @@ def base_exception_error(err): self.register_error_handler(AssertionError, assertion_error) self.register_error_handler(Exception, base_exception_error) + def settings_app(self, *args, **kwargs): + pass + def get_app(config=None, **kwargs): app = NewsroomNewsAPI(__name__, config=config, **kwargs) diff --git a/newsroom/upload.py b/newsroom/upload.py index 975081329..0db11e58b 100644 --- a/newsroom/upload.py +++ b/newsroom/upload.py @@ -78,5 +78,6 @@ def init_app(app): "authentication": None, "mongo_prefix": newsroom.MONGO_PREFIX, "internal_resource": True, + "mongo_indexes": {}, }, ) diff --git a/newsroom/web/factory.py b/newsroom/web/factory.py index 895adf49a..8e0bdf351 100644 --- a/newsroom/web/factory.py +++ b/newsroom/web/factory.py @@ -162,7 +162,6 @@ def _setup_theme(self): self.add_url_rule( self.static_url_path.replace("static", "theme") + "/", endpoint="theme", - host=self.static_host, view_func=self.send_theme_file, ) diff --git a/requirements.txt b/requirements.txt index ca6e4bb01..9f519c675 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ honcho>=1.0.1 gunicorn>=20.0.4,<22.1 PyRTF3>=0.47.5 xhtml2pdf>=0.2.4 -sentry-sdk[flask]>=1.5.7,<2.1 +sentry-sdk[flask]>=1.5.7,<2.12 eve-elastic>=7.3.1,<7.5 MarkupSafe<2.2 python3-saml>=1.15,<1.17 @@ -22,5 +22,5 @@ xmlsec==1.3.13 # pin xmlsec due to https://github.com/xmlsec/python-xmlsec/issu # https://github.com/xhtml2pdf/xhtml2pdf/issues/589 reportlab>=3.6.11,<4.3 -superdesk-core @ git+https://github.com/superdesk/superdesk-core.git@v2.7.0-rc4 -superdesk-planning @ git+https://github.com/superdesk/superdesk-planning.git@v2.7.0-rc1 +superdesk-core @ git+https://github.com/superdesk/superdesk-core.git@v2.7.3 +superdesk-planning @ git+https://github.com/superdesk/superdesk-planning.git@v2.7.2