diff --git a/README.md b/README.md index 3d6289e..491ca7f 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ Argostimè is a web-app to keep an eye on prices in webshops. The name is derived from Argos, the mythical giant with a hundred eyes, and "timè", Greek for price. -Argostimè is based on Flask and SQLAlchemy and has a modular structure so new shops can -be added easily. +Argostimè is requires Python 3.10 (or later) and is based on Flask and SQLAlchemy. +Argostimè has a modular structure so new shops can be added easily. The "official" version of Argostimè is available at [argostime.mrtijn.nl](https://argostime.mrtijn.nl/). diff --git a/argostime/__init__.py b/argostime/__init__.py index 8ca7d12..2cbbfc1 100644 --- a/argostime/__init__.py +++ b/argostime/__init__.py @@ -33,10 +33,9 @@ import configparser from flask import Flask +from flask_sqlalchemy import SQLAlchemy -from argostime.products import * -from argostime.exceptions import * -from argostime.models import * +db: SQLAlchemy = SQLAlchemy() def get_current_commit() -> str: """Return the hexadecimal hash of the current running commit.""" @@ -71,6 +70,7 @@ def create_app(): app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False app.config["GIT_CURRENT_COMMIT"] = get_current_commit() + db.init_app(app) with app.app_context(): diff --git a/argostime/crawler/shop/ah.py b/argostime/crawler/shop/ah.py index 7f4030c..c27e6ae 100644 --- a/argostime/crawler/shop/ah.py +++ b/argostime/crawler/shop/ah.py @@ -113,7 +113,7 @@ def crawl_ah(url: str) -> CrawlResult: "p", attrs={ "class" :lambda x: x and x.startswith("promo-sticker-text") } ) - + if len(promo_text_matches) == 0: promo_text_matches = soup.find_all( "div", diff --git a/argostime/graphs.py b/argostime/graphs.py index 44fdf3c..93da577 100644 --- a/argostime/graphs.py +++ b/argostime/graphs.py @@ -23,10 +23,10 @@ along with Argostimè. If not, see . """ -from typing import List, Tuple from datetime import datetime, timedelta import json +from argostime import db from argostime.exceptions import NoEffectivePriceAvailableException from argostime.models import ProductOffer, Price @@ -36,13 +36,16 @@ def generate_price_graph_data(offer: ProductOffer) -> str: time of a specific ProductOffer """ - prices: List[Price] = Price.query.filter_by( - product_offer_id=offer.id).order_by(Price.datetime).all() + prices = db.session.scalars( + db.select(Price) + .where(Price.product_offer_id == offer.id) + .order_by(Price.datetime) + ).all() - dates: List[datetime] = [] - effective_prices: List[float] = [] - sales_index: List[Tuple[int, int]] = [] - sales_dates: List[Tuple[datetime, datetime]] = [] + dates: list[datetime] = [] + effective_prices: list[float] = [] + sales_index: list[tuple[int, int]] = [] + sales_dates: list[tuple[datetime, datetime]] = [] index = 0 for price in prices: @@ -55,11 +58,11 @@ def generate_price_graph_data(offer: ProductOffer) -> str: sales_index.append((index, index)) else: sales_index[-1] = (sales_index[-1][0], index) - + index += 1 except NoEffectivePriceAvailableException: pass - + for sale in sales_index: start: datetime end: datetime @@ -68,7 +71,7 @@ def generate_price_graph_data(offer: ProductOffer) -> str: start = dates[sale[0]] - timedelta(hours=12) else: start = dates[sale[0]] - (dates[sale[0]] - dates[sale[0]-1]) / 2 - + if sale[1] == len(dates)-1: end = dates[sale[1]] + timedelta(hours=12) else: diff --git a/argostime/models.py b/argostime/models.py index e58ad31..8f195a9 100644 --- a/argostime/models.py +++ b/argostime/models.py @@ -28,15 +28,12 @@ from sys import maxsize from typing import List -from flask_sqlalchemy import SQLAlchemy - from argostime.crawler import crawl_url, CrawlResult from argostime.exceptions import CrawlerException, WebsiteNotImplementedException from argostime.exceptions import PageNotFoundException from argostime.exceptions import NoEffectivePriceAvailableException -db: SQLAlchemy = SQLAlchemy() - +from argostime import db class Webshop(db.Model): # type: ignore """A webshop, which may offer products.""" @@ -107,6 +104,11 @@ class ProductOffer(db.Model): # type: ignore db.ForeignKey("Webshop.id", ondelete="CASCADE"), nullable=False) url = db.Column(db.Unicode(1024), unique=True, nullable=False) time_added = db.Column(db.DateTime) + average_price = db.Column(db.Float) + minimum_price = db.Column(db.Float) + maximum_price = db.Column(db.Float) + # TODO: Memoize current price with reference to the most recent Price entry + prices = db.relationship("Price", backref="product_offer", lazy=True, cascade="all, delete", passive_deletes=True) @@ -116,30 +118,71 @@ def __str__(self): def get_current_price(self) -> Price: """Get the latest Price object related to this offer.""" - return Price.query.filter_by(product_offer_id=self.id).order_by(Price.datetime.desc()).first() - def get_average_price(self) -> float: - """Calculate the average price of this offer.""" + price = db.session.scalar( + db.select(Price) + .where(Price.product_offer_id == self.id) + .order_by(Price.datetime.desc()) + .limit(1) + ) + + return price + + def update_average_price(self) -> float: + """Calculate the average price of this offer and update ProductOffer.average_price.""" + logging.debug("Updating average price for %s", self) effective_price_values: List[float] = [] - for price in Price.query.filter_by(product_offer_id=self.id).all(): + + prices = db.session.scalars( + db.select(Price) + .where(Price.product_offer_id == self.id) + ).all() + + for price in prices: try: effective_price_values.append(price.get_effective_price()) except NoEffectivePriceAvailableException: # Ignore price entries without a valid price in calculating the price. pass try: - return statistics.mean(effective_price_values) + avg: float = statistics.mean(effective_price_values) + self.average_price = avg + db.session.commit() + return avg except statistics.StatisticsError: logging.debug("Called get_average_price for %s but no prices were found...", str(self)) return -1 + def get_average_price(self) -> float: + """Stub for new .average_price attribute + + DEPRECATED: Use ProductOffer.average_price instead. + """ + return self.average_price + + def get_prices_since(self, since_time: datetime) -> list[Price]: + """Get all prices since given date""" + prices_since = db.session.scalars( + db.select(Price) + .where(Price.product_offer_id == self.id) + .where(Price.datetime >= since_time) + ).all() + + prices_since_list: list[Price] = [] + for price in prices_since: + prices_since_list.append(price) + + return prices_since_list + def get_lowest_price_since(self, since_time: datetime) -> float: """Return the lowest effective price of this offer since a specific time.""" + logging.debug("Calculating lowest price since %s for %s", since_time, self) min_price: float = maxsize price: Price - for price in Price.query.filter( - Price.product_offer_id == self.id, - Price.datetime >= since_time).all(): + + prices_since = self.get_prices_since(since_time) + + for price in prices_since: try: if price.get_effective_price() < min_price: min_price = price.get_effective_price() @@ -149,17 +192,29 @@ def get_lowest_price_since(self, since_time: datetime) -> float: return min_price + def update_minimum_price(self) -> None: + """Update the minimum price ever in the minimum column""" + + min_price: float = self.get_lowest_price_since(self.time_added) + self.minimum_price = min_price + db.session.commit() + def get_lowest_price(self) -> float: - """Return the lowest effective price of this offer.""" - return self.get_lowest_price_since(self.time_added) + """Return the lowest effective price of this offer. + + DEPRECATED: Use ProductOffer.minimum_price instead + """ + return self.minimum_price def get_highest_price_since(self, since_time: datetime) -> float: """Return the highest effective price of this offer since a specific time.""" + logging.debug("Calculating highest price since %s for %s", since_time, self) max_price: float = -1 price: Price - for price in Price.query.filter( - Price.product_offer_id == self.id, - Price.datetime >= since_time).all(): + + prices_since = self.get_prices_since(since_time) + + for price in prices_since: try: if price.get_effective_price() > max_price: max_price = price.get_effective_price() @@ -168,19 +223,28 @@ def get_highest_price_since(self, since_time: datetime) -> float: return max_price + def update_maximum_price(self) -> None: + """Update the maximum price ever in the maximum_price column""" + + max_price: float = self.get_highest_price_since(self.time_added) + self.maximum_price = max_price + db.session.commit() + def get_highest_price(self) -> float: - """Return the highest effective price of this offer.""" - return self.get_highest_price_since(self.time_added) + """Return the highest effective price of this offer. + + DEPRECATED: Use ProductOffer.maximum_price instead. + """ + return self.maximum_price def get_price_standard_deviation_since(self, since_time: datetime) -> float: """Return the standard deviation of the effective price of this offer since a given date.""" effective_prices: List[float] = [] price: Price - for price in Price.query.filter( - Price.product_offer_id == self.id, - Price.datetime >= since_time).all(): + prices_since = self.get_prices_since(since_time) + for price in prices_since: try: effective_prices.append(price.get_effective_price()) except NoEffectivePriceAvailableException: @@ -195,6 +259,13 @@ def get_price_standard_deviation(self) -> float: """Return the standard deviation of the effective price of this offer.""" return self.get_price_standard_deviation_since(self.time_added) + def update_memoized_values(self) -> None: + """Update all memoized columns""" + + self.update_average_price() + self.update_minimum_price() + self.update_maximum_price() + def crawl_new_price(self) -> None: """Crawl the current price if we haven't already checked today.""" latest_price: Price = self.get_current_price() @@ -234,3 +305,5 @@ def crawl_new_price(self) -> None: ) db.session.add(price) db.session.commit() + + self.update_memoized_values() diff --git a/argostime/products.py b/argostime/products.py index c5fc935..20a341c 100644 --- a/argostime/products.py +++ b/argostime/products.py @@ -28,8 +28,9 @@ from typing import Tuple import urllib.parse +from argostime import db from argostime.exceptions import WebsiteNotImplementedException -from argostime.models import Webshop, Price, Product, ProductOffer, db +from argostime.models import Webshop, Price, Product, ProductOffer from argostime.crawler import crawl_url, CrawlResult, enabled_shops class ProductOfferAddResult(Enum): @@ -51,7 +52,10 @@ def add_product_offer_from_url(url: str) -> Tuple[ProductOfferAddResult, Product except KeyError as exception: raise WebsiteNotImplementedException(url) from exception - shop: Webshop = Webshop.query.filter(Webshop.hostname.contains(shop_info["hostname"])).first() + shop: Webshop = db.session.scalar( + db.select(Webshop) + .where(Webshop.hostname.contains(shop_info["hostname"])) + ) # Add Webshop if it can't be found in the database if shop is None: @@ -60,14 +64,20 @@ def add_product_offer_from_url(url: str) -> Tuple[ProductOfferAddResult, Product db.session.commit() # Check if this ProductOffer already exists - product_offer: ProductOffer = ProductOffer.query.filter_by(url=url).first() + product_offer: ProductOffer = db.session.scalar( + db.select(ProductOffer) + .where(ProductOffer.url == url) + ) if product_offer is not None: return (ProductOfferAddResult.ALREADY_EXISTS, product_offer) parse_results: CrawlResult = crawl_url(url) # Check if this Product already exists, otherwise add it to the database - product: Product = Product.query.filter_by(product_code=parse_results.product_code).first() + product: Product = db.session.scalar( + db.select(Product) + .where(Product.product_code == parse_results.product_code) + ) if product is None: product = Product( name=parse_results.product_name, @@ -99,5 +109,6 @@ def add_product_offer_from_url(url: str) -> Tuple[ProductOfferAddResult, Product ) db.session.add(price) db.session.commit() + offer.update_memoized_values() return (ProductOfferAddResult.ADDED, offer) diff --git a/argostime/routes.py b/argostime/routes.py index 6da650a..3c1aa44 100644 --- a/argostime/routes.py +++ b/argostime/routes.py @@ -31,6 +31,7 @@ from flask import render_template, abort, request, redirect from flask import Response +from argostime import db from argostime.exceptions import CrawlerException from argostime.exceptions import PageNotFoundException from argostime.exceptions import WebsiteNotImplementedException @@ -39,6 +40,7 @@ from argostime.products import ProductOfferAddResult, add_product_offer_from_url def add_product_url(url): + """Helper function for adding a product""" try: res, offer = add_product_offer_from_url(url) except WebsiteNotImplementedException: @@ -75,31 +77,49 @@ def index(): if request.method == "POST": return add_product_url(request.form["url"]) else: - products = Product.query.order_by(Product.id.desc()).limit(5).all() - discounts = Price.query.filter( - Price.datetime >= datetime.now().date(), - Price.on_sale == True # pylint: disable=C0121 + recently_added_products = db.session.scalars( + db.select(Product).order_by(Product.id.desc()).limit(5) + ).all() + + # TODO: Maybe join on productoffer & product? + discounts = db.session.scalars( + db.select(Price).where( + Price.datetime >= datetime.now().date(), + Price.on_sale == True # pylint: disable=C0121 + ) ).all() discounts.sort(key=lambda x: x.product_offer.product.name) - shops = Webshop.query.order_by(Webshop.name).all() + shops = db.session.scalars( + db.select(Webshop) + .order_by(Webshop.name) + ).all() return render_template( "index.html.jinja", - products=products, + products=recently_added_products, discounts=discounts, shops=shops) @app.route("/product/") def product_page(product_code): """Show the page for a specific product, with all known product offers""" - product: Product = Product.query.filter_by(product_code=product_code).first() + + product: Product = db.session.scalars( + db.select(Product) + .where(Product.product_code == product_code) + ).first() + + logging.debug("Rendering product page for %s based on product code %s", product, product_code) if product is None: abort(404) - offers: List[ProductOffer] = ProductOffer.query.filter_by( - product_id=product.id).join(Webshop).order_by(Webshop.name).all() + offers: List[ProductOffer] = db.session.scalars( + db.select(ProductOffer) + .where(ProductOffer.product_id == product.id) + .join(Webshop).order_by(Webshop.name) + ).all() return render_template( "product.html.jinja", @@ -109,7 +129,10 @@ def product_page(product_code): @app.route("/productoffer//price_step_graph_data.json") def offer_price_json(offer_id): """Generate the price step graph data of a specific offer""" - offer: ProductOffer = ProductOffer.query.get(offer_id) + offer: ProductOffer = db.session.execute( + db.select(ProductOffer) + .where(ProductOffer.id == offer_id) + ).scalar_one() if offer is None: abort(404) @@ -122,11 +145,14 @@ def all_offers(): """Generate an overview of all available offers""" show_variance: bool = False - if request.args.get("variance") != None: + if request.args.get("variance") is not None: show_variance = True - offers: List[ProductOffer] = ProductOffer.query.join( - Product).order_by(Product.name).all() + offers: List[ProductOffer] = db.session.scalars( + db.select(ProductOffer) + .join(Product) + .order_by(Product.name) + ).all() current_prices: Dict[ProductOffer, Price] = {} for offer in offers: @@ -142,17 +168,24 @@ def all_offers(): @app.route("/shop/") def webshop_page(shop_id): """Show a page with all the product offers of a specific webshop""" - shop: Webshop = Webshop.query.get(shop_id) + shop: Webshop = db.session.scalar( + db.select(Webshop) + .where(Webshop.id == shop_id) + ) if shop is None: abort(404) show_variance: bool = False - if request.args.get("variance") != None: + if request.args.get("variance") is not None: show_variance = True - offers: List[ProductOffer] = ProductOffer.query.filter_by( - shop_id=shop_id).join(Product).order_by(Product.name).all() + offers: List[ProductOffer] = db.session.scalars( + db.select(ProductOffer) + .where(ProductOffer.shop_id == shop_id) + .join(Product) + .order_by(Product.name) + ).all() current_prices: Dict[ProductOffer, Price] = {} for offer in offers: diff --git a/argostime/templates/all_offers.html.jinja b/argostime/templates/all_offers.html.jinja index 5de4b30..1d5e7ae 100644 --- a/argostime/templates/all_offers.html.jinja +++ b/argostime/templates/all_offers.html.jinja @@ -29,9 +29,9 @@ {% else %} {{ "€%.2f" | format(current_prices[offer].normal_price) }} ({{ current_prices[offer].datetime.strftime("%Y-%m-%d") }}) {% endif %} - {{ "€%.2f" | format(offer.get_average_price()) }} - {{ "€%.2f" | format(offer.get_lowest_price()) }} - {{ "€%.2f" | format(offer.get_highest_price()) }} + {{ "€%.2f" | format(offer.average_price) }} + {{ "€%.2f" | format(offer.minimum_price) }} + {{ "€%.2f" | format(offer.maximum_price) }} {% if show_variance %} {{ "%.2f" | format(offer.get_price_standard_deviation()) }} {% endif %} diff --git a/argostime/templates/product.html.jinja b/argostime/templates/product.html.jinja index 1993d2c..e3d5756 100644 --- a/argostime/templates/product.html.jinja +++ b/argostime/templates/product.html.jinja @@ -23,9 +23,9 @@ {% else %} {{ "€%.2f" | format(offer.get_current_price().normal_price) }} ({{ offer.get_current_price().datetime.strftime("%Y-%m-%d") }}) {% endif %} - {{ "€%.2f" | format(offer.get_average_price()) }} - {{ "€%.2f" | format(offer.get_lowest_price()) }} - {{ "€%.2f" | format(offer.get_highest_price()) }} + {{ "€%.2f" | format(offer.average_price) }} + {{ "€%.2f" | format(offer.minimum_price) }} + {{ "€%.2f" | format(offer.maximum_price) }} {{ "%.2f" | format(offer.get_price_standard_deviation()) }} {{ offer.time_added.strftime("%Y-%m-%d") }} diff --git a/argostime/templates/shop.html.jinja b/argostime/templates/shop.html.jinja index a8facea..707d867 100644 --- a/argostime/templates/shop.html.jinja +++ b/argostime/templates/shop.html.jinja @@ -29,9 +29,9 @@ {% else %} {{ "€%.2f" | format(current_prices[offer].normal_price) }} ({{ current_prices[offer].datetime.strftime("%Y-%m-%d") }}) {% endif %} - {{ "€%.2f" | format(offer.get_average_price()) }} - {{ "€%.2f" | format(offer.get_lowest_price()) }} - {{ "€%.2f" | format(offer.get_highest_price()) }} + {{ "€%.2f" | format(offer.average_price) }} + {{ "€%.2f" | format(offer.minimum_price) }} + {{ "€%.2f" | format(offer.maximum_price) }} {% if show_variance %} {{ "%.2f" | format(offer.get_price_standard_deviation()) }} {% endif %} diff --git a/argostime_update_prices.py b/argostime_update_prices.py old mode 100755 new mode 100644 index b323e68..d1f2aeb --- a/argostime_update_prices.py +++ b/argostime_update_prices.py @@ -24,37 +24,31 @@ import random import logging -from multiprocessing import Process import time -from argostime.models import ProductOffer, Webshop -from argostime import create_app +from argostime.models import ProductOffer +from argostime import create_app, db app = create_app() app.app_context().push() -def update_shop_offers(shop_id: int) -> None: - """Crawl all the offers of one shop""" +initial_sleep_time: float = random.uniform(0, 600) +logging.debug("Sleeping for %f seconds", initial_sleep_time) +time.sleep(initial_sleep_time) - offer: ProductOffer - for offer in ProductOffer.query.filter_by(shop_id=shop_id).all(): - logging.info("Crawling %s", str(offer)) +offers = db.session.scalars( + db.select(ProductOffer) +).all() - try: - offer.crawl_new_price() - except Exception as exception: - logging.error("Received %s while updating price of %s, continuing...", exception, offer) +offer: ProductOffer +for offer in offers: + logging.info("Crawling %s", str(offer)) - next_sleep_time: float = random.uniform(1, 180) - logging.debug("Sleeping for %f seconds", next_sleep_time) - time.sleep(next_sleep_time) + try: + offer.crawl_new_price() + except Exception as exception: + logging.error("Received %s while updating price of %s, continuing...", exception, offer) -if __name__ == "__main__": - for shop in Webshop.query.all(): - shop_process: Process = Process( - target=update_shop_offers, - args=[shop.id], - name=f"ShopProcess({shop.id})") - - logging.info("Starting process %s", shop_process) - shop_process.start() + next_sleep_time: float = random.uniform(1, 180) + logging.debug("Sleeping for %f seconds", next_sleep_time) + time.sleep(next_sleep_time) diff --git a/argostime_update_prices_parallel.py b/argostime_update_prices_parallel.py new file mode 100755 index 0000000..1bbedd1 --- /dev/null +++ b/argostime_update_prices_parallel.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +""" + argostime_update_prices.py + + Standalone script to update prices in the database. + + Copyright (c) 2022, 2023 Martijn + + This file is part of Argostimè. + + Argostimè is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Argostimè is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Argostimè. If not, see . +""" + +import random +import logging +from multiprocessing import Process +import time + +from argostime.models import ProductOffer, Webshop +from argostime import create_app, db + +app = create_app() +app.app_context().push() + +def update_shop_offers(shop_id: int) -> None: + """Crawl all the offers of one shop""" + + offers: list[ProductOffer] = db.session.scalars( + db.select(ProductOffer) + .where(ProductOffer.shop_id == shop_id) + ).all() + + offer: ProductOffer + for offer in offers: + logging.info("Crawling %s", str(offer)) + + try: + offer.crawl_new_price() + except Exception as exception: + logging.error("Received %s while updating price of %s, continuing...", exception, offer) + + next_sleep_time: float = random.uniform(1, 180) + logging.debug("Sleeping for %f seconds", next_sleep_time) + time.sleep(next_sleep_time) + +if __name__ == "__main__": + + shops: list[Webshop] = db.session.scalars( + db.select(Webshop) + .order_by(Webshop.id) + ).all() + + for shop in shops: + shop_process: Process = Process( + target=update_shop_offers, + args=[shop.id], + name=f"ShopProcess({shop.id})") + + logging.info("Starting process %s", shop_process) + shop_process.start() diff --git a/create_indexes.py b/create_indexes.py new file mode 100755 index 0000000..4ef5794 --- /dev/null +++ b/create_indexes.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +""" + create_indexes.py + + Standalone script to add indexes in the database. + + Copyright (c) 2023 Martijn + + This file is part of Argostimè. + + Argostimè is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Argostimè is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Argostimè. If not, see . +""" +import logging + +from sqlalchemy import text +from sqlalchemy.exc import OperationalError + +from argostime import create_app, db +from argostime.models import ProductOffer, Product, Price, Webshop + +app = create_app() +app.app_context().push() + +logging.info("Adding indexes") + +indexes = [ + db.Index("idx_Price_datetime", Price.datetime), + db.Index("idx_Price_product_offer", Price.product_offer_id), + db.Index("idx_Price_product_offer_id_datetime", Price.product_offer_id, Price.datetime), + db.Index("idx_ProductOffer_shop_id", ProductOffer.shop_id), + db.Index("idx_ProductOffer_product_id", ProductOffer.product_id), + db.Index("idx_Webshop_hostname", Webshop.hostname), + db.Index("idx_Product_product_code", Product.product_code), + +] + +for index in indexes: + try: + index.create(db.engine) + except OperationalError as e: + logging.error("%s", e) diff --git a/manual_update.py b/manual_update.py index 3d79c27..8498f7a 100755 --- a/manual_update.py +++ b/manual_update.py @@ -25,20 +25,19 @@ import sys import logging +from argostime import create_app, db from argostime.models import ProductOffer -from argostime import create_app app = create_app() app.app_context().push() - try: product_offer_id: int = int(sys.argv[1]) except: print("No number given") sys.exit(-1) -offer: ProductOffer = ProductOffer.query.get(product_offer_id) +offer: ProductOffer = db.session.execute(db.select(ProductOffer).where(ProductOffer.id == product_offer_id)).scalar_one() logging.debug("Found offer %s", product_offer_id) logging.debug("Manually updating ProductOffer %s", offer) diff --git a/migration_add_productoffer_avg_price_column.py b/migration_add_productoffer_avg_price_column.py new file mode 100755 index 0000000..4a22ade --- /dev/null +++ b/migration_add_productoffer_avg_price_column.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +""" + argostime_update_prices.py + + Standalone script to add new columns added in the sqlalchemy-v2 branch. + + Copyright (c) 2023 Martijn + + This file is part of Argostimè. + + Argostimè is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Argostimè is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Argostimè. If not, see . +""" + +import logging + +from sqlalchemy import text +from sqlalchemy.exc import OperationalError + +from argostime import create_app, db +from argostime.models import ProductOffer, Product + +app = create_app() +app.app_context().push() + +logging.info("Adding average_price column") + +try: + db.session.execute(text('ALTER TABLE ProductOffer ADD COLUMN average_price float')) +except OperationalError: + logging.info("Column already seems to exist, fine") + +try: + db.session.execute(text('ALTER TABLE ProductOffer ADD COLUMN minimum_price float')) +except OperationalError: + logging.info("Column already seems to exist, fine") + +try: + db.session.execute(text('ALTER TABLE ProductOffer ADD COLUMN maximum_price float')) +except OperationalError: + logging.info("Column already seems to exist, fine") + +logging.info("Calculate average prices") + +offers = db.session.scalars( + db.select(ProductOffer) + .join(Product) + .order_by(Product.name) +).all() + +offer: ProductOffer +for offer in offers: + logging.info("Calculating initial memoization values for %s", offer) + offer.update_memoized_values() diff --git a/pyproject.toml b/pyproject.toml index ba88fb2..bc77bd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,24 @@ +[project] +name = "Argostimè" +description = "Keep an eye on prices" +readme = "README.md" +requires-python = ">=3.10" +license = {file = "LICENSE"} +classifiers = [ + "Framework :: Flask", + "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", + "Natural Language :: Dutch", + "Operating System :: POSIX :: Linux", +] + +[project.urls] +Homepage = "https://github.com/m-rtijn/argostime" + [build-system] requires = [ "requests >= 2.27.1", "beautifulsoup4 >= 4.10.0", "Flask-SQLAlchemy >= 2.5.1", + "SQLAlchemy >= 2", + "gunicorn" ] diff --git a/requirements.txt b/requirements.txt index 25f255b..4a0adae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ requests>=2.27.1 beautifulsoup4>=4.10.0 Flask-SQLAlchemy>=2.5.1 gunicorn -SQLAlchemy >= 1.4.46, < 2 +SQLAlchemy >= 2 diff --git a/requirements_development.txt b/requirements_development.txt index 3a68abe..a84e007 100644 --- a/requirements_development.txt +++ b/requirements_development.txt @@ -1,2 +1,5 @@ -mypy -pylint +-r requirements.txt + +mypy >= 1.3.0 +pylint >= 2.17.4 +types-requests >= 2.31.0.1 \ No newline at end of file