diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d41c964 --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2025 RandomProgramm3r + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 7f48d63..c2f7676 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ -# Steam Market Scraper Parser +# Steam Market Parser [![Linting](https://github.com/RandomProgramm3r/Steam-Market-Scraper/actions/workflows/linting.yml/badge.svg)](https://github.com/RandomProgramm3r/Steam-Market-Scraper/actions) +[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) [Steam Community Market](https://steamcommunity.com/market/) ## TOC @@ -15,6 +17,8 @@ - [🧩 Usage](#-usage) - [🔨 Function Signature](#-function-signature) - [📤 Example](#-example) + - [🔁 Synchronous usage](#-synchronous-usage) + - [⚡ Asynchronous usage](#-asynchronous-usage) ## 📋 Description @@ -77,13 +81,13 @@ For code consistency and quality checks, use Ruff - a unified linter/formatter: ```bash # Run linting checks. -ruff check . +ruff check # Auto-fix fixable lint issues -ruff check . --fix +ruff check --fix # Format code. -ruff format . +ruff format ``` @@ -115,12 +119,12 @@ def market_scraper( ## 📤 Example +### 🔁 Synchronous usage ```python import data import scraper # Example usage: Fetch price information for 'Dreams & Nightmares Case' in USD for the CS2 app. -# see more in examples.py print( scraper.market_scraper( 'Dreams & Nightmares Case', @@ -131,10 +135,58 @@ print( ``` #### Output json data: ```json +{ + "success": true, + "lowest_price": "$2.19", + "volume": "112,393", + "median_price": "$2.16", + "game_name": "CS2", + "currency_name": "USD", + "item_name": "Dreams & Nightmares Case" +} +``` + +### ⚡ Asynchronous usage + +#### see more in examples.py +```python +async def main(): + items = [ + ('Dreams & Nightmares Case', data.steam_data.Apps.CS2.value, data.steam_data.Currency.USD.value), + ('Mann Co. Supply Crate Key', data.steam_data.Apps.TEAM_FORTRESS_2.value, data.steam_data.Currency.EUR.value), + ... + ] + + tasks = [ + src.scraper.async_.market_scraper_async(name, app_id, currency) + for name, app_id, currency in items + ] + results = await asyncio.gather(*tasks) + for result in results: + print(result) + +if __name__ == '__main__': + asyncio.run(main()) +``` + +#### Output json data: +```json +{ + "success": true, + "lowest_price": "$2.19", + "volume": "112,393", + "median_price": "$2.16", + "game_name": "CS2", + "currency_name": "USD", + "item_name": "Dreams & Nightmares Case" +} { "success": true, - "lowest_price": "$1.90", - "volume": "77,555", - "median_price": "$1.90" + "lowest_price": "2,01€", + "volume": "18,776", + "median_price": "2,03€", + "game_name": "TEAM_FORTRESS_2", + "currency_name": "EUR", + "item_name": "Mann Co. Supply Crate Key" } ``` \ No newline at end of file diff --git a/data/__init__.py b/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data.py b/data/steam_data.py similarity index 100% rename from data.py rename to data/steam_data.py diff --git a/examples.py b/examples.py index 59ad9d5..d1f4b25 100644 --- a/examples.py +++ b/examples.py @@ -1,39 +1,57 @@ -import data -import scraper +import asyncio -if __name__ == '__main__': - print( - scraper.market_scraper( +import data.steam_data +import src.scraper.async_ +import src.scraper.sync + +# sync +print( + src.scraper.sync.market_scraper_sync( + 'Dreams & Nightmares Case', + data.steam_data.Apps.CS2.value, + data.steam_data.Currency.USD.value, + ), +) + + +# async +async def main(): + items = [ + ( 'Dreams & Nightmares Case', - data.Apps.CS2.value, - data.Currency.USD.value, + data.steam_data.Apps.CS2.value, + data.steam_data.Currency.USD.value, ), - ) - print( - scraper.market_scraper( + ( 'Mann Co. Supply Crate Key', - data.Apps.TEAM_FORTRESS_2.value, - data.Currency.EUR.value, + data.steam_data.Apps.TEAM_FORTRESS_2.value, + data.steam_data.Currency.EUR.value, ), - ) - print( - scraper.market_scraper( + ( 'Doomsday Hoodie', - data.Apps.PUBG.value, - data.Currency.GBP.value, + data.steam_data.Apps.PUBG.value, + data.steam_data.Currency.GBP.value, ), - ) - print( - scraper.market_scraper( + ( 'AWP | Neo-Noir (Factory New)', - data.Apps.CS2.value, - data.Currency.USD.value, + data.steam_data.Apps.CS2.value, + data.steam_data.Currency.USD.value, ), - ) - print( - scraper.market_scraper( + ( 'Snowcamo Jacket', - data.Apps.RUST.value, - data.Currency.CHF.value, + data.steam_data.Apps.RUST.value, + data.steam_data.Currency.CHF.value, ), - ) + ] + + tasks = [ + src.scraper.async_.market_scraper_async(name, app_id, currency) + for name, app_id, currency in items + ] + results = await asyncio.gather(*tasks) + for result in results: + print(result) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/requirements.txt b/requirements.txt index 7143f75..ffcc7ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -ruff==0.11.6 \ No newline at end of file +aiohttp==3.11.18 +ruff==0.11.10 \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/scraper/__init__.py b/src/scraper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/scraper/async_.py b/src/scraper/async_.py new file mode 100644 index 0000000..ed4c447 --- /dev/null +++ b/src/scraper/async_.py @@ -0,0 +1,79 @@ +import asyncio +import json + +import aiohttp + +import data.steam_data +import src.scraper.common + + +async def fetch_price( + session: aiohttp.ClientSession, + item_name: str, + app_id: int, + currency: int, +) -> str: + """ + A request is sent asynchronously to the Steam Market and a formatted + JSON response or error text is returned. + """ + encoded = src.scraper.common.encode_item_name(item_name) + url = ( + f'{src.scraper.common.BASE_URL}' + f'appid={app_id}' + f'¤cy={currency}' + f'&market_hash_name={encoded}' + ) + + try: + async with session.get(url, timeout=5) as response: + text = await response.text() + parsed = json.loads(text) + + return src.scraper.common.format_response( + item_name, + app_id, + currency, + parsed, + ) + + except aiohttp.ClientResponseError as e: + return f'Server error: {e.status}' + except aiohttp.ClientConnectionError: + return 'Network error. Please check your internet connection.' + except asyncio.TimeoutError: + return 'Timeout error.' + except json.JSONDecodeError: + return 'Error decoding JSON response from server.' + + +async def market_scraper_async( + item_name: str, + app_id: int, + currency: int = data.steam_data.Currency.USD.value, +) -> str: + """ + Asynchronously fetch the market price for a given Steam item. + + This coroutine validates the input parameters, opens an HTTP session, + and delegates to the `fetch_price` helper to retrieve live pricing data + from the Steam market. + + Parameters: + item_name (str): The exact name of the Steam marketplace item. + app_id (int): The Steam application ID where the item is listed. + currency (int, optional): The currency code for price conversion. + Defaults to USD if not provided. + + Returns: + str: A formatted result, which may be a JSON string + (via `format_response`) or an error message if the fetch fails. + + Raises: + ValueError: If any of `item_name`, `app_id`, or `currency` is invalid. + aiohttp.ClientError: If the HTTP request to the Steam API fails. + """ + + src.scraper.common.validate_parameters(item_name, app_id, currency) + async with aiohttp.ClientSession() as session: + return await fetch_price(session, item_name, app_id, currency) diff --git a/scraper.py b/src/scraper/common.py similarity index 61% rename from scraper.py rename to src/scraper/common.py index f8754cb..57e03f3 100644 --- a/scraper.py +++ b/src/scraper/common.py @@ -1,10 +1,10 @@ import enum import json -import urllib.error import urllib.parse -import urllib.request -import data +import data.steam_data + +BASE_URL = 'https://steamcommunity.com/market/priceoverview/?' def validate_enum_value( @@ -75,8 +75,8 @@ def validate_parameters(item_name: str, app_id: int, currency: int) -> None: if not item_name.strip(): raise ValueError('Item name cannot consist solely of whitespace.') - validate_enum_value(app_id, data.Apps, 'app ID') - validate_enum_value(currency, data.Currency, 'currency') + validate_enum_value(app_id, data.steam_data.Apps, 'app ID') + validate_enum_value(currency, data.steam_data.Currency, 'currency') def encode_item_name(item_name: str) -> str: @@ -97,52 +97,45 @@ def encode_item_name(item_name: str) -> str: return urllib.parse.quote_plus(item_name.strip()).replace('~', '%7E') -def market_scraper( +def format_response( item_name: str, app_id: int, - currency: int = data.Currency.USD.value, + currency: int, + result: dict | str, ) -> str: """ - Retrieves data from the Steam Community Market for the specified item. + Construct a standardized response string for the Steam market data. - The function validates the input parameters, encodes the item name, - constructs the URL, and performs an HTTP request. It returns a formatted - JSON response or an error message. + This function builds a JSON-formatted string when `result` is a dictionary, + or returns the raw string otherwise. It enriches the output by inserting + the game name, currency name, and item name into the result payload if + they are not already present. Args: - item_name (str): The full name of the item. - app_id (int): The application ID for the request. - currency (int, optional): The currency code. - Defaults to data.Currency.USD.value. (USD) + item_name (str): The human-readable name of the item being queried. + app_id (int): The Steam application ID associated with the item. + currency (int): Numeric code representing the currency (per Steam API). + result (dict | str): The price data or error message. If a dict, + it will be pretty-printed as JSON; if a string, it will be returned + unchanged. Returns: - str: A formatted JSON response or an error message. - """ - validate_parameters(item_name, app_id, currency) - - encoded_name = encode_item_name(item_name) + str: A JSON-formatted string including `game_name`, `currency_name`, + `item_name`, and any other data present in `result`, or the raw + string if `result` is not a dict. - base_url = 'https://steamcommunity.com/market/priceoverview/?' - - url = ( - f'{base_url}' - f'appid={app_id}' - f'¤cy={currency}' - f'&market_hash_name={encoded_name}' - ) - - try: - with urllib.request.urlopen(url, timeout=5) as response: - result = json.loads(response.read().decode()) + Raises: + KeyError: If expected fields in the Steam data lookup are missing. + TypeError: If `result` is not of type dict or str. + """ - if isinstance(result, dict): - return json.dumps(result, indent=4, ensure_ascii=False) + game_name = data.steam_data.Apps(app_id).name + currency_name = data.steam_data.Currency(currency).name - return result + if isinstance(result, dict): + result.setdefault('game_name', game_name) + result.setdefault('currency_name', currency_name) + result.setdefault('item_name', item_name) + return json.dumps(result, indent=4, ensure_ascii=False) - except urllib.error.HTTPError as e: - return f'Server error: {e.code}' - except urllib.error.URLError: - return 'Network error. Please check your internet connection.' - except json.JSONDecodeError: - return 'Error decoding JSON response from server.' + return result diff --git a/src/scraper/sync.py b/src/scraper/sync.py new file mode 100644 index 0000000..abeaca5 --- /dev/null +++ b/src/scraper/sync.py @@ -0,0 +1,56 @@ +import json +import urllib.error +import urllib.request + +import data.steam_data +import src.scraper.common + + +def market_scraper_sync( + item_name: str, + app_id: int, + currency: int = data.steam_data.Currency.USD.value, +) -> str: + """ + Retrieves data from the Steam Community Market for the specified item. + + The function validates the input parameters, encodes the item name, + constructs the URL, and performs an HTTP request. It returns a formatted + JSON response or an error message. + + Args: + item_name (str): The full name of the item. + app_id (int): The application ID for the request. + currency (int, optional): The currency code. + Defaults to data.Currency.USD.value. (USD) + + Returns: + str: A formatted JSON response or an error message. + """ + src.scraper.common.validate_parameters(item_name, app_id, currency) + + encoded_name = src.scraper.common.encode_item_name(item_name) + + url = ( + f'{src.scraper.common.BASE_URL}' + f'appid={app_id}' + f'¤cy={currency}' + f'&market_hash_name={encoded_name}' + ) + + try: + with urllib.request.urlopen(url, timeout=5) as response: + parsed = json.loads(response.read().decode()) + return src.scraper.common.format_response( + item_name, + app_id, + currency, + parsed, + ) + + except urllib.error.HTTPError as e: + return f'Server error: {e.code}' + except urllib.error.URLError: + return 'Network error. Please check your internet connection.' + except json.JSONDecodeError: + return 'Error decoding JSON response from server.' diff --git a/test_market_scraper.py b/test_market_scraper.py index 88cfd91..2dcff37 100644 --- a/test_market_scraper.py +++ b/test_market_scraper.py @@ -1,80 +1,127 @@ import json import unittest -import data -import scraper +import data.steam_data +import src.scraper.common +import src.scraper.sync + + +class TestValidateEnumValue(unittest.TestCase): + def test_non_int_raises_type_error(self): + with self.assertRaises(TypeError): + src.scraper.common.validate_enum_value( + 'not_int', + data.steam_data.Currency, + 'currency', + ) + + def test_invalid_value_raises_value_error(self): + with self.assertRaises(ValueError): + src.scraper.common.validate_enum_value( + 99999, + data.steam_data.Currency, + 'currency', + ) + + def test_valid_enum_value_passes(self): + src.scraper.common.validate_enum_value( + data.steam_data.Currency.USD.value, + data.steam_data.Currency, + 'currency', + ) class TestEncodeItemName(unittest.TestCase): def test_encode_item_name_regular(self): item_name = 'Dreams & Nightmares' expected = 'Dreams+%26+Nightmares' - self.assertEqual(scraper.encode_item_name(item_name), expected) + self.assertEqual( + src.scraper.common.encode_item_name(item_name), + expected, + ) def test_encode_item_name_tilde(self): item_name = 'Music Kit | Chipzel, ~Yellow Magic~' expected = 'Music+Kit+%7C+Chipzel%2C+%7EYellow+Magic%7E' - self.assertEqual(scraper.encode_item_name(item_name), expected) + self.assertEqual( + src.scraper.common.encode_item_name(item_name), + expected, + ) def test_encode_item_with_leading_and_trailing_whitespaces(self): item_name = ' AWP | Neo-Noir (Factory New) ' expected = 'AWP+%7C+Neo-Noir+%28Factory+New%29' - self.assertEqual(scraper.encode_item_name(item_name), expected) + self.assertEqual( + src.scraper.common.encode_item_name(item_name), + expected, + ) + + def test_special_chars_encoding(self): + item = 'A+B/C: D' + encoded = src.scraper.common.encode_item_name(item) + self.assertIn('%2B', encoded) + self.assertIn('%2F', encoded) + self.assertIn('%3A', encoded) + + def test_internal_whitespace_preserved(self): + item = 'Alpha Beta' + encoded = src.scraper.common.encode_item_name(item) + self.assertEqual(encoded, 'Alpha++++Beta') def test_encode_item_name_empty(self): - self.assertEqual(scraper.encode_item_name(''), '') + self.assertEqual(src.scraper.common.encode_item_name(''), '') def test_encode_item_name_none(self): with self.assertRaises(AttributeError): - scraper.encode_item_name(None) + src.scraper.common.encode_item_name(None) class ValidateParameters(unittest.TestCase): def test_market_scraper_invalid_item_name_type(self): with self.assertRaises(TypeError): - scraper.market_scraper( + src.scraper.common.validate_parameters( 11111, - data.Apps.CS2.value, - data.Currency.USD.value, + data.steam_data.Apps.CS2.value, + data.steam_data.Currency.USD.value, ) def test_market_scraper_invalid_app_id_type(self): with self.assertRaises(TypeError): - scraper.validate_parameters( + src.scraper.common.validate_parameters( 'Dreams & Nightmares Case', 'CS2', - data.Currency.USD.value, + data.steam_data.Currency.USD.value, ) def test_market_scraper_invalid_currency_type(self): with self.assertRaises(TypeError): - scraper.validate_parameters( + src.scraper.common.validate_parameters( 'Dreams & Nightmares Case', - data.Apps.CS2.value, + data.steam_data.Apps.CS2.value, 'USD', ) def test_market_scraper_item_name_consist_solely_of_whitespace(self): with self.assertRaises(ValueError): - scraper.market_scraper( + src.scraper.sync.market_scraper_sync( ' ', - data.Apps.CS2.value, - data.Currency.USD.value, + data.steam_data.Apps.CS2.value, + data.steam_data.Currency.USD.value, ) def test_market_scraper_invalid_app_id(self): with self.assertRaises(ValueError): - scraper.market_scraper( + src.scraper.sync.market_scraper_sync( 'Dreams & Nightmares Case', 00000, - data.Currency.USD.value, + data.steam_data.Currency.USD.value, ) def test_market_scraper_invalid_currency(self): with self.assertRaises(ValueError): - scraper.market_scraper( + src.scraper.sync.market_scraper_sync( 'Dreams & Nightmares Case', - data.Apps.CS2.value, + data.steam_data.Apps.CS2.value, 00000, ) @@ -82,10 +129,10 @@ def test_market_scraper_invalid_currency(self): class TestMarketScraper(unittest.TestCase): def test_valid_response_keys(self): response = json.loads( - scraper.market_scraper( + src.scraper.sync.market_scraper_sync( 'Dreams & Nightmares Case', - data.Apps.CS2.value, - data.Currency.USD.value, + data.steam_data.Apps.CS2.value, + data.steam_data.Currency.USD.value, ), ) expected_keys = {'success', 'lowest_price', 'volume', 'median_price'} @@ -93,10 +140,10 @@ def test_valid_response_keys(self): def test_invalid_game_for_selected_item(self): response = json.loads( - scraper.market_scraper( + src.scraper.sync.market_scraper_sync( 'Dreams & Nightmares Case', - data.Apps.PUBG.value, - data.Currency.USD.value, + data.steam_data.Apps.PUBG.value, + data.steam_data.Currency.USD.value, ), ) expected_key = {'success'} @@ -105,5 +152,28 @@ def test_invalid_game_for_selected_item(self): self.assertFalse(unexpected_keys.issubset(response.keys())) +class TestValidateParametersBoundary(unittest.TestCase): + def test_long_item_name(self): + long_name = 'A' * 1000 + src.scraper.common.validate_parameters( + long_name, + data.steam_data.Apps.CS2.value, + data.steam_data.Currency.USD.value, + ) + + def test_whitespace_vs_valid(self): + with self.assertRaises(ValueError): + src.scraper.common.validate_parameters( + ' ', + data.steam_data.Apps.CS2.value, + data.steam_data.Currency.USD.value, + ) + src.scraper.common.validate_parameters( + ' x ', + data.steam_data.Apps.CS2.value, + data.steam_data.Currency.USD.value, + ) + + if __name__ == '__main__': unittest.main()