")
def webshop_page(shop_id):
"""Show a page with all the product offers of a specific webshop"""
- shop: Webshop = Webshop.query.get(shop_id)
+ shop: Webshop = db.session.scalar(
+ db.select(Webshop)
+ .where(Webshop.id == shop_id)
+ )
if shop is None:
abort(404)
show_variance: bool = False
- if request.args.get("variance") != None:
+ if request.args.get("variance") is not None:
show_variance = True
- offers: List[ProductOffer] = ProductOffer.query.filter_by(
- shop_id=shop_id).join(Product).order_by(Product.name).all()
+ offers: List[ProductOffer] = db.session.scalars(
+ db.select(ProductOffer)
+ .where(ProductOffer.shop_id == shop_id)
+ .join(Product)
+ .order_by(Product.name)
+ ).all()
current_prices: Dict[ProductOffer, Price] = {}
for offer in offers:
diff --git a/argostime/templates/all_offers.html.jinja b/argostime/templates/all_offers.html.jinja
index 5de4b30..1d5e7ae 100644
--- a/argostime/templates/all_offers.html.jinja
+++ b/argostime/templates/all_offers.html.jinja
@@ -29,9 +29,9 @@
{% else %}
{{ "€%.2f" | format(current_prices[offer].normal_price) }} ({{ current_prices[offer].datetime.strftime("%Y-%m-%d") }}) |
{% endif %}
- {{ "€%.2f" | format(offer.get_average_price()) }} |
- {{ "€%.2f" | format(offer.get_lowest_price()) }} |
- {{ "€%.2f" | format(offer.get_highest_price()) }} |
+ {{ "€%.2f" | format(offer.average_price) }} |
+ {{ "€%.2f" | format(offer.minimum_price) }} |
+ {{ "€%.2f" | format(offer.maximum_price) }} |
{% if show_variance %}
{{ "%.2f" | format(offer.get_price_standard_deviation()) }} |
{% endif %}
diff --git a/argostime/templates/product.html.jinja b/argostime/templates/product.html.jinja
index 1993d2c..e3d5756 100644
--- a/argostime/templates/product.html.jinja
+++ b/argostime/templates/product.html.jinja
@@ -23,9 +23,9 @@
{% else %}
{{ "€%.2f" | format(offer.get_current_price().normal_price) }} ({{ offer.get_current_price().datetime.strftime("%Y-%m-%d") }}) |
{% endif %}
- {{ "€%.2f" | format(offer.get_average_price()) }} |
- {{ "€%.2f" | format(offer.get_lowest_price()) }} |
- {{ "€%.2f" | format(offer.get_highest_price()) }} |
+ {{ "€%.2f" | format(offer.average_price) }} |
+ {{ "€%.2f" | format(offer.minimum_price) }} |
+ {{ "€%.2f" | format(offer.maximum_price) }} |
{{ "%.2f" | format(offer.get_price_standard_deviation()) }} |
{{ offer.time_added.strftime("%Y-%m-%d") }} |
diff --git a/argostime/templates/shop.html.jinja b/argostime/templates/shop.html.jinja
index a8facea..707d867 100644
--- a/argostime/templates/shop.html.jinja
+++ b/argostime/templates/shop.html.jinja
@@ -29,9 +29,9 @@
{% else %}
{{ "€%.2f" | format(current_prices[offer].normal_price) }} ({{ current_prices[offer].datetime.strftime("%Y-%m-%d") }}) |
{% endif %}
- {{ "€%.2f" | format(offer.get_average_price()) }} |
- {{ "€%.2f" | format(offer.get_lowest_price()) }} |
- {{ "€%.2f" | format(offer.get_highest_price()) }} |
+ {{ "€%.2f" | format(offer.average_price) }} |
+ {{ "€%.2f" | format(offer.minimum_price) }} |
+ {{ "€%.2f" | format(offer.maximum_price) }} |
{% if show_variance %}
{{ "%.2f" | format(offer.get_price_standard_deviation()) }} |
{% endif %}
diff --git a/argostime_update_prices.py b/argostime_update_prices.py
old mode 100755
new mode 100644
index b323e68..d1f2aeb
--- a/argostime_update_prices.py
+++ b/argostime_update_prices.py
@@ -24,37 +24,31 @@
import random
import logging
-from multiprocessing import Process
import time
-from argostime.models import ProductOffer, Webshop
-from argostime import create_app
+from argostime.models import ProductOffer
+from argostime import create_app, db
app = create_app()
app.app_context().push()
-def update_shop_offers(shop_id: int) -> None:
- """Crawl all the offers of one shop"""
+initial_sleep_time: float = random.uniform(0, 600)
+logging.debug("Sleeping for %f seconds", initial_sleep_time)
+time.sleep(initial_sleep_time)
- offer: ProductOffer
- for offer in ProductOffer.query.filter_by(shop_id=shop_id).all():
- logging.info("Crawling %s", str(offer))
+offers = db.session.scalars(
+ db.select(ProductOffer)
+).all()
- try:
- offer.crawl_new_price()
- except Exception as exception:
- logging.error("Received %s while updating price of %s, continuing...", exception, offer)
+offer: ProductOffer
+for offer in offers:
+ logging.info("Crawling %s", str(offer))
- next_sleep_time: float = random.uniform(1, 180)
- logging.debug("Sleeping for %f seconds", next_sleep_time)
- time.sleep(next_sleep_time)
+ try:
+ offer.crawl_new_price()
+ except Exception as exception:
+ logging.error("Received %s while updating price of %s, continuing...", exception, offer)
-if __name__ == "__main__":
- for shop in Webshop.query.all():
- shop_process: Process = Process(
- target=update_shop_offers,
- args=[shop.id],
- name=f"ShopProcess({shop.id})")
-
- logging.info("Starting process %s", shop_process)
- shop_process.start()
+ next_sleep_time: float = random.uniform(1, 180)
+ logging.debug("Sleeping for %f seconds", next_sleep_time)
+ time.sleep(next_sleep_time)
diff --git a/argostime_update_prices_parallel.py b/argostime_update_prices_parallel.py
new file mode 100755
index 0000000..1bbedd1
--- /dev/null
+++ b/argostime_update_prices_parallel.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""
+ argostime_update_prices.py
+
+ Standalone script to update prices in the database.
+
+ Copyright (c) 2022, 2023 Martijn
+
+ This file is part of Argostimè.
+
+ Argostimè is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ Argostimè is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with Argostimè. If not, see .
+"""
+
+import random
+import logging
+from multiprocessing import Process
+import time
+
+from argostime.models import ProductOffer, Webshop
+from argostime import create_app, db
+
+app = create_app()
+app.app_context().push()
+
+def update_shop_offers(shop_id: int) -> None:
+ """Crawl all the offers of one shop"""
+
+ offers: list[ProductOffer] = db.session.scalars(
+ db.select(ProductOffer)
+ .where(ProductOffer.shop_id == shop_id)
+ ).all()
+
+ offer: ProductOffer
+ for offer in offers:
+ logging.info("Crawling %s", str(offer))
+
+ try:
+ offer.crawl_new_price()
+ except Exception as exception:
+ logging.error("Received %s while updating price of %s, continuing...", exception, offer)
+
+ next_sleep_time: float = random.uniform(1, 180)
+ logging.debug("Sleeping for %f seconds", next_sleep_time)
+ time.sleep(next_sleep_time)
+
+if __name__ == "__main__":
+
+ shops: list[Webshop] = db.session.scalars(
+ db.select(Webshop)
+ .order_by(Webshop.id)
+ ).all()
+
+ for shop in shops:
+ shop_process: Process = Process(
+ target=update_shop_offers,
+ args=[shop.id],
+ name=f"ShopProcess({shop.id})")
+
+ logging.info("Starting process %s", shop_process)
+ shop_process.start()
diff --git a/create_indexes.py b/create_indexes.py
new file mode 100755
index 0000000..4ef5794
--- /dev/null
+++ b/create_indexes.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+"""
+ create_indexes.py
+
+ Standalone script to add indexes in the database.
+
+ Copyright (c) 2023 Martijn
+
+ This file is part of Argostimè.
+
+ Argostimè is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ Argostimè is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with Argostimè. If not, see .
+"""
+import logging
+
+from sqlalchemy import text
+from sqlalchemy.exc import OperationalError
+
+from argostime import create_app, db
+from argostime.models import ProductOffer, Product, Price, Webshop
+
+app = create_app()
+app.app_context().push()
+
+logging.info("Adding indexes")
+
+indexes = [
+ db.Index("idx_Price_datetime", Price.datetime),
+ db.Index("idx_Price_product_offer", Price.product_offer_id),
+ db.Index("idx_Price_product_offer_id_datetime", Price.product_offer_id, Price.datetime),
+ db.Index("idx_ProductOffer_shop_id", ProductOffer.shop_id),
+ db.Index("idx_ProductOffer_product_id", ProductOffer.product_id),
+ db.Index("idx_Webshop_hostname", Webshop.hostname),
+ db.Index("idx_Product_product_code", Product.product_code),
+
+]
+
+for index in indexes:
+ try:
+ index.create(db.engine)
+ except OperationalError as e:
+ logging.error("%s", e)
diff --git a/manual_update.py b/manual_update.py
index 3d79c27..8498f7a 100755
--- a/manual_update.py
+++ b/manual_update.py
@@ -25,20 +25,19 @@
import sys
import logging
+from argostime import create_app, db
from argostime.models import ProductOffer
-from argostime import create_app
app = create_app()
app.app_context().push()
-
try:
product_offer_id: int = int(sys.argv[1])
except:
print("No number given")
sys.exit(-1)
-offer: ProductOffer = ProductOffer.query.get(product_offer_id)
+offer: ProductOffer = db.session.execute(db.select(ProductOffer).where(ProductOffer.id == product_offer_id)).scalar_one()
logging.debug("Found offer %s", product_offer_id)
logging.debug("Manually updating ProductOffer %s", offer)
diff --git a/migration_add_productoffer_avg_price_column.py b/migration_add_productoffer_avg_price_column.py
new file mode 100755
index 0000000..4a22ade
--- /dev/null
+++ b/migration_add_productoffer_avg_price_column.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+"""
+ argostime_update_prices.py
+
+ Standalone script to add new columns added in the sqlalchemy-v2 branch.
+
+ Copyright (c) 2023 Martijn
+
+ This file is part of Argostimè.
+
+ Argostimè is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ Argostimè is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with Argostimè. If not, see .
+"""
+
+import logging
+
+from sqlalchemy import text
+from sqlalchemy.exc import OperationalError
+
+from argostime import create_app, db
+from argostime.models import ProductOffer, Product
+
+app = create_app()
+app.app_context().push()
+
+logging.info("Adding average_price column")
+
+try:
+ db.session.execute(text('ALTER TABLE ProductOffer ADD COLUMN average_price float'))
+except OperationalError:
+ logging.info("Column already seems to exist, fine")
+
+try:
+ db.session.execute(text('ALTER TABLE ProductOffer ADD COLUMN minimum_price float'))
+except OperationalError:
+ logging.info("Column already seems to exist, fine")
+
+try:
+ db.session.execute(text('ALTER TABLE ProductOffer ADD COLUMN maximum_price float'))
+except OperationalError:
+ logging.info("Column already seems to exist, fine")
+
+logging.info("Calculate average prices")
+
+offers = db.session.scalars(
+ db.select(ProductOffer)
+ .join(Product)
+ .order_by(Product.name)
+).all()
+
+offer: ProductOffer
+for offer in offers:
+ logging.info("Calculating initial memoization values for %s", offer)
+ offer.update_memoized_values()
diff --git a/pyproject.toml b/pyproject.toml
index ba88fb2..bc77bd6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,24 @@
+[project]
+name = "Argostimè"
+description = "Keep an eye on prices"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {file = "LICENSE"}
+classifiers = [
+ "Framework :: Flask",
+ "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+ "Natural Language :: Dutch",
+ "Operating System :: POSIX :: Linux",
+]
+
+[project.urls]
+Homepage = "https://github.com/m-rtijn/argostime"
+
[build-system]
requires = [
"requests >= 2.27.1",
"beautifulsoup4 >= 4.10.0",
"Flask-SQLAlchemy >= 2.5.1",
+ "SQLAlchemy >= 2",
+ "gunicorn"
]
diff --git a/requirements.txt b/requirements.txt
index 25f255b..4a0adae 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,4 @@ requests>=2.27.1
beautifulsoup4>=4.10.0
Flask-SQLAlchemy>=2.5.1
gunicorn
-SQLAlchemy >= 1.4.46, < 2
+SQLAlchemy >= 2
diff --git a/requirements_development.txt b/requirements_development.txt
index 3a68abe..a84e007 100644
--- a/requirements_development.txt
+++ b/requirements_development.txt
@@ -1,2 +1,5 @@
-mypy
-pylint
+-r requirements.txt
+
+mypy >= 1.3.0
+pylint >= 2.17.4
+types-requests >= 2.31.0.1
\ No newline at end of file