Skip to content

Commit

Permalink
Tweaks (#47)
Browse files Browse the repository at this point in the history 
* Added handler for 403 messages

* Added missing fields alert

* fixed tests
  • Loading branch information
@angusgoody
angusgoody authored Aug 14, 2023
1 parent 11d0b61 commit 11e5c66
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 18 deletions.
12 changes: 11 additions & 1 deletion app/Http/Controllers/ProductScraperController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public function scrapeProduct(Request $request): JsonResponse

// Try to connect 3 times before giving up
$response = Http::timeout(3)->withHeaders([
'User-Agent' => "MyWishlistApp lookup",
'User-Agent' => "Wish4it",
])->retry(2, 1000)->get($url);


Expand All @@ -60,8 +60,18 @@ public function scrapeProduct(Request $request): JsonResponse

}
catch (RequestException $exception) {
$statusCode = $exception->response->status();
if ($statusCode == 403) {
Log::channel("scraper")->warning('403 Forbidden error when scraping', [
'url' => $url,
'error_message' => $exception->getMessage(),
]);
return response()->json(['error' => 'The website has blocked us from retrieving product details :('], 403);
}

Log::channel("scraper")->error('A request error occurred during URL scraping', [
'url' => $url,
'status_code' => $statusCode,
'error_message' => $exception->getMessage(),
]);

Expand Down
1 change: 1 addition & 0 deletions app/Services/Product.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public function toArray(): array
"brand" => $this->brand,
"price" => $this->price,
"image" => $this->image,
"hasMissedFields" => count($this->getMissingFields()) > 0
];
}

Expand Down
24 changes: 17 additions & 7 deletions resources/js/Components/wishlist/NewItemModal.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,17 +67,24 @@

<!-- Details screen -->
<div v-else class="flex-col gap-y-10">

<!-- Missing fields -->
<div v-if="hasMissedFields" class="flex my-3 items-center p-4 text-sm text-gray-800 rounded-lg bg-gray-50 dark:bg-gray-800 dark:text-gray-300" role="alert">
<svg class="flex-shrink-0 inline w-4 h-4 mr-3" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="currentColor" viewBox="0 0 20 20">
<path d="M10 .5a9.5 9.5 0 1 0 9.5 9.5A9.51 9.51 0 0 0 10 .5ZM9.5 4a1.5 1.5 0 1 1 0 3 1.5 1.5 0 0 1 0-3ZM12 15H8a1 1 0 0 1 0-2h1v-3H8a1 1 0 0 1 0-2h2a1 1 0 0 1 1 1v4h1a1 1 0 0 1 0 2Z"/>
</svg>
<span class="sr-only">Info</span>
<div>
<p>Some fields could not be retrieved automatically </p>
</div>
</div>

<!-- Image -->
<div v-if="form.image" class="w-full mt-5">
<div class="w-24 h-24 rounded-md overflow-hidden mx-auto bg-white">
<img :src="form.image" alt="Your Image" class="w-full h-full object-contain">
</div>





</div>

<!-- Form -->
Expand Down Expand Up @@ -121,7 +128,6 @@
id="price"
type="text"
class="w-full mt-1"
placeholder="5"
/>
<InputError v-if="form.errors.price" class="mt-1" :message="form.errors.price" />
</div>
Expand Down Expand Up @@ -215,6 +221,7 @@ const urlForm = useForm({
let isOpen = ref(props.open)
let showDetails = ref(false)
let hasMissedFields = ref(false)
let showLoadUrlButton = ref(false)
let loadUrlError = ref(false);
Expand Down Expand Up @@ -277,10 +284,11 @@ function sendUrl(){
form.brand = product.brand
form.price = product.price
form.url = urlForm.url
showDetails.value = true;
form.image = product.image
hasMissedFields.value = product.hasMissedFields
showDetails.value = true;
}
console.log(response)
})
.catch(error => {
Expand Down Expand Up @@ -340,6 +348,8 @@ function reset(){
form.errors = {};
urlForm.errors = {};
hasMissedFields.value = false;
loadUrlError.value = false;
form.reset();
urlForm.reset();
Expand Down
30 changes: 20 additions & 10 deletions tests/Feature/ProductScraperTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ public function test_hoodie_scrape(): void
"name" => "Balatapa Hood",
"brand" => "Idioma",
"price" => "89.00",
"image" => "//idioma.world/cdn/shop/products/Balatapa_Hood_Grey_Full_Hung_Web_grande.jpg?v=1679829385"
"image" => "//idioma.world/cdn/shop/products/Balatapa_Hood_Grey_Full_Hung_Web_grande.jpg?v=1679829385",
"hasMissedFields" => false
];

// Assert specific values
Expand All @@ -56,7 +57,8 @@ public function test_john_lewis_scrape()
"name" => "Bialetti Moka Express Hob Espresso Coffee Maker, 1 Cup",
"brand" => "Bialetti",
"price" => "24.00",
"image" => "http://johnlewis.scene7.com/is/image/JohnLewis/001407520"
"image" => "http://johnlewis.scene7.com/is/image/JohnLewis/001407520",
"hasMissedFields" => false
];

// Assert specific values
Expand Down Expand Up @@ -84,7 +86,8 @@ public function test_go_outdoors_scrape()
"name" => "Men’s Boreas GTX Mid Walking Boots",
"brand" => "Scarpa",
"price" => "190.00",
"image" => "https://i1.adis.ws/i/jpl/bl_16243260_a"
"image" => "https://i1.adis.ws/i/jpl/bl_16243260_a",
"hasMissedFields" => false
];

// Assert specific values
Expand Down Expand Up @@ -113,7 +116,8 @@ public function test_apple_store_scrape()
"name" => "iPhone 14 Pro 256GB Silver",
"brand" => null,
"price" => "1209.00",
"image" => "https://store.storeimages.cdn-apple.com/4668/as-images.apple.com/is/iphone-14-pro-finish-select-202209-6-1inch-silver?wid=2560&hei=1440&fmt=jpeg&qlt=95&.v=1663703840488"
"image" => "https://store.storeimages.cdn-apple.com/4668/as-images.apple.com/is/iphone-14-pro-finish-select-202209-6-1inch-silver?wid=2560&hei=1440&fmt=jpeg&qlt=95&.v=1663703840488",
"hasMissedFields" => true
];

// Assert specific values
Expand Down Expand Up @@ -141,7 +145,8 @@ public function test_ebay_scrape()
"name" => "UK Delivery | Herman Miller Sayl Chairs | Black Frame &amp; Base | Blue Seat",
"brand" => "Herman Miller",
"price" => "289.00",
"image" => "https://i.ebayimg.com/images/g/YjAAAOSwR0Rklf8P/s-l1600.jpg"
"image" => "https://i.ebayimg.com/images/g/YjAAAOSwR0Rklf8P/s-l1600.jpg",
"hasMissedFields" => false
];

// Assert specific values
Expand Down Expand Up @@ -170,7 +175,8 @@ public function test_argos_scrape()
"name" => "Sony Full Frame A7mk3 Camera with SEL2870 Lens ",
"brand" => null,
"price" => "1699.99",
"image" => null
"image" => null,
"hasMissedFields" => true
];

// Assert specific values
Expand All @@ -197,7 +203,8 @@ public function test_smaller_website_scrape()
"name" => "Canon RF 1.4x Extender",
"brand" => null,
"price" => "579.00",
"image" => "https://harrison-cameras.s3.amazonaws.com/p/s/4113C005AA.jpg"
"image" => "https://harrison-cameras.s3.amazonaws.com/p/s/4113C005AA.jpg",
"hasMissedFields" => true
];

// Assert specific values
Expand All @@ -221,7 +228,8 @@ public function test_bandq_scrape()
"name" => "GoodHome Denia Brown Wooden 2 seater Square Table",
"brand" => null,
"price" => "34.00",
"image" => "https://media.diy.com/is/image/Kingfisher/goodhome-denia-brown-wooden-2-seater-square-table~3663602936015_01bq?\$MOB_PREV$&\$width=768&\$height=768"
"image" => "https://media.diy.com/is/image/Kingfisher/goodhome-denia-brown-wooden-2-seater-square-table~3663602936015_01bq?\$MOB_PREV$&\$width=768&\$height=768",
"hasMissedFields" => true
];

// Assert specific values
Expand All @@ -248,7 +256,8 @@ public function test_amazon_scrape()
"name" => "Duracell 2032 Twin Pack - silver",
"brand" => null,
"price" => "4.30",
"image" => "https://m.media-amazon.com/images/I/71iAp6vtJeL.__AC_SX300_SY300_QL70_ML2_.jpg"
"image" => "https://m.media-amazon.com/images/I/71iAp6vtJeL.__AC_SX300_SY300_QL70_ML2_.jpg",
"hasMissedFields" => true
];

// Assert specific values
Expand All @@ -275,7 +284,8 @@ public function test_etsy_scrape()
"name" => "Babies personalised Easter Bunny plush toy",
"brand" => "LulabayUK",
"price" => "12.00",
"image" => "https://i.etsystatic.com/28485446/c/3000/2384/0/153/il/92de6f/4089068291/il_340x270.4089068291_imz3.jpg"
"image" => "https://i.etsystatic.com/28485446/c/3000/2384/0/153/il/92de6f/4089068291/il_340x270.4089068291_imz3.jpg",
"hasMissedFields" => false
];

// Assert specific values
Expand Down

0 comments on commit 11e5c66

Please sign in to comment.