From 29792ea98b49fcf60c368e98a654dca018bc13b9 Mon Sep 17 00:00:00 2001 From: Illia Date: Fri, 30 Sep 2022 14:15:05 +0300 Subject: [PATCH] fix: Code generation in README.md Paste documentation examples to `README.md` in a loop instead of duplicating the same lines for all of the engines. Also, changed signature of the `serpapi.Client` to accept three parameters instead of a dict. This mentally distinguishes arguments allowed to the client from the `Client#search()` method. Also, fixed `make all` execution by prepending `python -m` to Python commands. Co-authored-by: Dmitriy Zub --- Makefile | 10 +- README.md | 479 +++++++++--------- README.md.erb | 161 ++---- serpapi/serpapi.py | 93 ++-- tests/example_search_google_local_services.py | 13 +- 5 files changed, 342 insertions(+), 414 deletions(-) diff --git a/Makefile b/Makefile index 20b4823..e3a4d70 100644 --- a/Makefile +++ b/Makefile @@ -22,11 +22,11 @@ clean: # lint check lint: - pylint serpapi + python3 -m pylint serpapi # test with Python 3 test: - pytest --cov=serpapi --cov-report html tests/*.py + python3 -m pytest --cov=serpapi --cov-report html tests/*.py # install dependencies # @@ -58,12 +58,12 @@ oobt: build check: oobt - twine check ${dist} + python3 -m twine check ${dist} release: # check - twine upload ${dist} + python3 -m twine upload ${dist} # run example only # and display output (-s) example: - pytest -s "tests/test_example.py::TestExample::test_async" + python3 -m pytest -s "tests/test_example.py::TestExample::test_async" diff --git a/README.md b/README.md index bcba299..92a9d64 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # User guide [![SerpApi-Python](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml/badge.svg)](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml) -[SerpApi]](https://serpapi.com) allows to scrape any search engine results. - It's easy, fast, easy, feature rich, cost effective, scalable and reliable. +[SerpApi](https://serpapi.com) allows to scrape any search engine results. +It's easy, fast, easy, feature rich, cost effective, scalable and reliable. -This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi]](https://serpapi.com). +This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi](https://serpapi.com). This is an open source project hosted under https://github.com/serpapi/serpapi-python. -SerpApi.com provides a [script builder]](https://serpapi.com/demo) to get you started quickly. +SerpApi.com provides a [script builder](https://serpapi.com/demo) to get you started quickly. ## Installation SerpApi can be installed with pip. @@ -17,78 +17,73 @@ $ python -m pip install serpapi ``` ## Quick start -First things first, import the serpapi module: + +The following example runs a search for `"coffee"` using your secret API key which you can find at [SerpApi Dashboard](https://serpapi.com/manage-api-key) page. The `serpapi.search` object handles all of the details of connection pooling and thread safety so that you don't have to: ```python import serpapi -``` -You'll need a client instance to make a search. This object handles all of the details of connection pooling and thread safety so that you don't have to: -```python -client = serpapi.Client() -``` -To make a search using SerpApi.com: +client = serpapi.Client( + api_key = "secret_api_key", # from serpapi.com +) -```python -parameter = { - api_key: "secret_api_key", # from serpapi.com - engine: "google", # search engine - q: "coffee", # search topic - location: "Austin,TX" # location +parameters = { + "q": "coffee", } -results = searpapi.search(parameter) -``` -Putting everything together. -```python -import serpapi -parameter = { - api_key: "secret_api_key", # from serpapi.com - engine: "google", # search engine - q: "coffee", # search topic - location: "Austin,TX" # location -} -results = searpapi.search(parameter) +results = client.search(parameters) + print(results) ``` ### Advanced settings SerpApi Client uses urllib3 under the hood. -Optionally, rhe HTTP connection can be tuned: + +Optionally, the HTTP connection can be tuned: - timeout : connection timeout by default 60s - - retries : attempt to reconnect if the connection failed by default: False. - serpapi is reliable at 99.99% but your company network might not be as stable. - - ```python -parameter = { - retries: 5, - timeout: 4.0, - # extra user parameters -} + - retries : attempt to reconnect if the connection failed by default: False. Documentation: https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html#urllib3.util.Retry + - api_key : the secret user API available from http://serpapi.com/manage-api-key + + SerpApi's SLA is 99.95% successful searches: https://serpapi.com/faq/general/do-you-provide-sla-guarantees + +```python +client = serpapi.Client( + retries = 5, # or urllib3.util.Retry(connect=5, read=2) + timeout = 4.2, + api_key = "secret_api_key", # from serpapi.com +) ``` -for more details: [URL LIB3 documentation]](https://urllib3.readthedocs.io/en/stable/user-guide.html) +For more details: [`urllib3` documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html) ## Basic example per search engines -### Search Bing + + + +### Search Google Local Services ```python import serpapi import pprint import os + def test_search_google_local_services(self): client = serpapi.Client({ -'engine': 'bing', -'api_key': os.getenv("API_KEY") + 'api_key': os.getenv("API_KEY"), + 'timeout': 61.1, + 'retries': 2 }) data = client.search({ -'q': 'coffee', + 'engine': 'google_local_services', + 'q': 'Electrician', + 'place_id': 'ChIJOwg_06VPwokRYv534QaPC8g', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['local_ads']) ``` - test: tests/example_search_bing.py -see: [https://serpapi.com/bing-search-api](https://serpapi.com/bing-search-api) +test: tests/example_search_google_local_services.py +
+Documentation: https://serpapi.com/google-local-services-search-api + ### Search Baidu ```python @@ -97,407 +92,425 @@ import pprint import os client = serpapi.Client({ -'engine': 'baidu', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'baidu', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - test: tests/example_search_baidu.py -see: [https://serpapi.com/baidu-search-api](https://serpapi.com/baidu-search-api) +test: tests/example_search_baidu.py +
+Documentation: https://serpapi.com/baidu-search-api -### Search Yahoo + +### Search Ebay ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'yahoo', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'ebay', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'p': 'coffee', + '_nkw': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - test: tests/example_search_yahoo.py -see: [https://serpapi.com/yahoo-search-api](https://serpapi.com/yahoo-search-api) +test: tests/example_search_ebay.py +
+Documentation: https://serpapi.com/ebay-search-api -### Search Youtube + +### Search Apple App Store ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'youtube', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'apple_app_store', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'search_query': 'coffee', + 'term': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['video_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['organic_results']) ``` - test: tests/example_search_youtube.py -see: [https://serpapi.com/youtube-search-api](https://serpapi.com/youtube-search-api) +test: tests/example_search_apple_app_store.py +
+Documentation: https://serpapi.com/apple-app-store-search-api -### Search Walmart + +### Search Google Product ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'walmart', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_product', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'query': 'coffee', + 'q': 'coffee', + 'product_id': '4172129135583325756', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['product_results']) ``` - test: tests/example_search_walmart.py -see: [https://serpapi.com/walmart-search-api](https://serpapi.com/walmart-search-api) +test: tests/example_search_google_product.py +
+Documentation: https://serpapi.com/google-product-search-api -### Search Ebay + +### Search Google Maps ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'ebay', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_maps', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'_nkw': 'coffee', + 'q': 'pizza', + 'll': '@40.7455096,-74.0083012,15.1z', + 'type': 'search', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['local_results']) ``` - test: tests/example_search_ebay.py -see: [https://serpapi.com/ebay-search-api](https://serpapi.com/ebay-search-api) +test: tests/example_search_google_maps.py +
+Documentation: https://serpapi.com/google-maps-search-api -### Search Naver + +### Search Google Images ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'naver', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'query': 'coffee', + 'engine': 'google', + 'tbm': 'isch', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['ads_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['images_results']) ``` - test: tests/example_search_naver.py -see: [https://serpapi.com/naver-search-api](https://serpapi.com/naver-search-api) +test: tests/example_search_google_images.py +
+Documentation: https://serpapi.com/google-images-search-api -### Search Home Depot + +### Search Google Search ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'home_depot', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'table', + 'q': 'coffee', + 'engine': 'google', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['products']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['organic_results']) ``` - test: tests/example_search_home_depot.py -see: [https://serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api) +test: tests/example_search_google_search.py +
+Documentation: https://serpapi.com/google-search-search-api -### Search Apple App Store + +### Search Google Jobs ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'apple_app_store', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_jobs', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'term': 'coffee', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['jobs_results']) ``` - test: tests/example_search_apple_app_store.py -see: [https://serpapi.com/apple-app-store](https://serpapi.com/apple-app-store) +test: tests/example_search_google_jobs.py +
+Documentation: https://serpapi.com/google-jobs-search-api -### Search Duckduckgo + +### Search Naver ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'duckduckgo', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'naver', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'query': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['ads_results']) ``` - test: tests/example_search_duckduckgo.py -see: [https://serpapi.com/duckduckgo-search-api](https://serpapi.com/duckduckgo-search-api) +test: tests/example_search_naver.py +
+Documentation: https://serpapi.com/naver-search-api -### Search Google Search + +### Search Google Scholar ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'google', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_scholar', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', -'engine': 'google', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - test: tests/example_search_google_search.py -see: [https://serpapi.com/search-api](https://serpapi.com/search-api) +test: tests/example_search_google_scholar.py +
+Documentation: https://serpapi.com/google-scholar-search-api -### Search Google Scholar + +### Search Youtube ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'google_scholar', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'youtube', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'search_query': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['video_results']) ``` - test: tests/example_search_google_scholar.py -see: [https://serpapi.com/google-scholar-api](https://serpapi.com/google-scholar-api) +test: tests/example_search_youtube.py +
+Documentation: https://serpapi.com/youtube-search-api -### Search Google Autocomplete + +### Search Google Reverse Image ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'google_autocomplete', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_reverse_image', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'image_url': 'https://i.imgur.com/5bGzZi7.jpg', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['suggestions']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['image_sizes']) ``` - test: tests/example_search_google_autocomplete.py -see: [https://serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api) +test: tests/example_search_google_reverse_image.py +
+Documentation: https://serpapi.com/google-reverse-image-search-api -### Search Google Product + +### Search Yahoo ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'google_product', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'yahoo', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', -'product_id': '4172129135583325756', + 'p': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['product_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['organic_results']) ``` - test: tests/example_search_google_product.py -see: [https://serpapi.com/google-product-api](https://serpapi.com/google-product-api) +test: tests/example_search_yahoo.py +
+Documentation: https://serpapi.com/yahoo-search-api -### Search Google Reverse Image + +### Search Duckduckgo ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'google_reverse_image', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'duckduckgo', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'image_url': 'https://i.imgur.com/5bGzZi7.jpg', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['image_sizes']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['organic_results']) ``` - test: tests/example_search_google_reverse_image.py -see: [https://serpapi.com/google-reverse-image](https://serpapi.com/google-reverse-image) +test: tests/example_search_duckduckgo.py +
+Documentation: https://serpapi.com/duckduckgo-search-api -### Search Google Events + +### Search Bing ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'google_events', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'bing', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['events_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['organic_results']) ``` - test: tests/example_search_google_events.py -see: [https://serpapi.com/google-events-api](https://serpapi.com/google-events-api) +test: tests/example_search_bing.py +
+Documentation: https://serpapi.com/bing-search-api -### Search Google Local Services + +### Search Google Autocomplete ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'google_local_services', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_autocomplete', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'Electrician', -'place_id': 'ChIJOwg_06VPwokRYv534QaPC8g', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['local_ads']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['suggestions']) ``` - test: tests/example_search_google_local_services.py -see: [https://serpapi.com/google-local-services-api](https://serpapi.com/google-local-services-api) +test: tests/example_search_google_autocomplete.py +
+Documentation: https://serpapi.com/google-autocomplete-search-api -### Search Google Maps + +### Search Home Depot ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'google_maps', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'home_depot', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'pizza', -'ll': '@40.7455096,-74.0083012,15.1z', -'type': 'search', + 'q': 'table', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['local_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['products']) ``` - test: tests/example_search_google_maps.py -see: [https://serpapi.com/google-maps-api](https://serpapi.com/google-maps-api) +test: tests/example_search_home_depot.py +
+Documentation: https://serpapi.com/home-depot-search-api -### Search Google Jobs + +### Search Walmart ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'google_jobs', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'walmart', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'query': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['jobs_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['organic_results']) ``` - test: tests/example_search_google_jobs.py -see: [https://serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api) +test: tests/example_search_walmart.py +
+Documentation: https://serpapi.com/walmart-search-api -### Search Google Play + +### Search Google Events ```python import serpapi import pprint import os -def test_search_google_play(self): client = serpapi.Client({ -'engine': 'google_play', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_events', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'kite', -'store': 'apps', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['events_results']) ``` - test: tests/example_search_google_play.py -see: [https://serpapi.com/google-play-api](https://serpapi.com/google-play-api) +test: tests/example_search_google_events.py +
+Documentation: https://serpapi.com/google-events-search-api -### Search Google Images + +### Search Google Play ```python import serpapi import pprint import os + def test_search_google_play(self): client = serpapi.Client({ -'engine': 'google', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_play', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'engine': 'google', -'tbm': 'isch', -'q': 'coffee', + 'q': 'kite', + 'store': 'apps', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['images_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +pp.pprint(data['organic_results']) ``` - test: tests/example_search_google_images.py -see: [https://serpapi.com/images-results](https://serpapi.com/images-results) +test: tests/example_search_google_play.py +
+Documentation: https://serpapi.com/google-play-search-api # Developer Guide ### Key goals diff --git a/README.md.erb b/README.md.erb index 9e1fb9f..2f6a9ac 100644 --- a/README.md.erb +++ b/README.md.erb @@ -1,22 +1,22 @@ <%- def snippet(format, path, start, stop) slice = File.new(path).readlines[start..stop] - slice.reject! { |l| l =~ /self.assertIsNone\(/ } - buf = slice.map { |l| l.gsub(/(^\s+)/, '')}.join + slice.reject! { |l| l.match?(/self.assertIsNone\(/) || l.match?(/# os\.getenv\("API_KEY"\)/) } + buf = slice.map { |l| l.gsub(/(^\s{4})/, '').gsub(/^\s*$/, '') }.join buf.gsub!('self.assertIsNotNone(', "pp = pprint.PrettyPrinter(indent=2)\npp.pprint(") - %Q(```#{format}\nimport serpapi\nimport pprint\nimport os\n\n#{buf}```\n test: #{path}) + %Q(```#{format}\nimport serpapi\nimport pprint\nimport os\n\n#{buf}```\ntest: #{path}) end -%> # User guide [![SerpApi-Python](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml/badge.svg)](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml) -[SerpApi]](https://serpapi.com) allows to scrape any search engine results. - It's easy, fast, easy, feature rich, cost effective, scalable and reliable. +[SerpApi](https://serpapi.com) allows to scrape any search engine results. +It's easy, fast, easy, feature rich, cost effective, scalable and reliable. -This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi]](https://serpapi.com). +This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi](https://serpapi.com). This is an open source project hosted under https://github.com/serpapi/serpapi-python. -SerpApi.com provides a [script builder]](https://serpapi.com/demo) to get you started quickly. +SerpApi.com provides a [script builder](https://serpapi.com/demo) to get you started quickly. ## Installation SerpApi can be installed with pip. @@ -26,142 +26,55 @@ $ python -m pip install serpapi ``` ## Quick start -First things first, import the serpapi module: + +The following example runs a search for `"coffee"` using your secret API key which you can find at [SerpApi Dashboard](https://serpapi.com/manage-api-key) page. The `serpapi.search` object handles all of the details of connection pooling and thread safety so that you don't have to: ```python import serpapi -``` -You'll need a client instance to make a search. This object handles all of the details of connection pooling and thread safety so that you don't have to: -```python -client = serpapi.Client() -``` -To make a search using SerpApi.com: +client = serpapi.Client( + api_key = "secret_api_key", # from serpapi.com +) -```python -parameter = { - api_key: "secret_api_key", # from serpapi.com - engine: "google", # search engine - q: "coffee", # search topic - location: "Austin,TX" # location +parameters = { + "q": "coffee", } -results = searpapi.search(parameter) -``` -Putting everything together. -```python -import serpapi -parameter = { - api_key: "secret_api_key", # from serpapi.com - engine: "google", # search engine - q: "coffee", # search topic - location: "Austin,TX" # location -} -results = searpapi.search(parameter) +results = client.search(parameters) + print(results) ``` ### Advanced settings SerpApi Client uses urllib3 under the hood. -Optionally, rhe HTTP connection can be tuned: - - timeout : connection timeout by default 60s - - retries : attempt to reconnect if the connection failed by default: False. - serpapi is reliable at 99.99% but your company network might not be as stable. - - ```python -parameter = { - retries: 5, - timeout: 4.0, - # extra user parameters -} -``` - -for more details: [URL LIB3 documentation]](https://urllib3.readthedocs.io/en/stable/user-guide.html) - -## Basic example per search engines -### Search Bing -<%= snippet('python', 'tests/example_search_bing.py', 9, 25) %> -see: [https://serpapi.com/bing-search-api](https://serpapi.com/bing-search-api) - -### Search Baidu -<%= snippet('python', 'tests/example_search_baidu.py', 9, 25) %> -see: [https://serpapi.com/baidu-search-api](https://serpapi.com/baidu-search-api) -### Search Yahoo -<%= snippet('python', 'tests/example_search_yahoo.py', 9, 25) %> -see: [https://serpapi.com/yahoo-search-api](https://serpapi.com/yahoo-search-api) - -### Search Youtube -<%= snippet('python', 'tests/example_search_youtube.py', 9, 25) %> -see: [https://serpapi.com/youtube-search-api](https://serpapi.com/youtube-search-api) - -### Search Walmart -<%= snippet('python', 'tests/example_search_walmart.py', 9, 25) %> -see: [https://serpapi.com/walmart-search-api](https://serpapi.com/walmart-search-api) - -### Search Ebay -<%= snippet('python', 'tests/example_search_ebay.py', 9, 25) %> -see: [https://serpapi.com/ebay-search-api](https://serpapi.com/ebay-search-api) - -### Search Naver -<%= snippet('python', 'tests/example_search_naver.py', 9, 25) %> -see: [https://serpapi.com/naver-search-api](https://serpapi.com/naver-search-api) - -### Search Home Depot -<%= snippet('python', 'tests/example_search_home_depot.py', 9, 25) %> -see: [https://serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api) - -### Search Apple App Store -<%= snippet('python', 'tests/example_search_apple_app_store.py', 9, 25) %> -see: [https://serpapi.com/apple-app-store](https://serpapi.com/apple-app-store) - -### Search Duckduckgo -<%= snippet('python', 'tests/example_search_duckduckgo.py', 9, 25) %> -see: [https://serpapi.com/duckduckgo-search-api](https://serpapi.com/duckduckgo-search-api) - -### Search Google Search -<%= snippet('python', 'tests/example_search_google_search.py', 9, 25) %> -see: [https://serpapi.com/search-api](https://serpapi.com/search-api) - -### Search Google Scholar -<%= snippet('python', 'tests/example_search_google_scholar.py', 9, 25) %> -see: [https://serpapi.com/google-scholar-api](https://serpapi.com/google-scholar-api) - -### Search Google Autocomplete -<%= snippet('python', 'tests/example_search_google_autocomplete.py', 9, 25) %> -see: [https://serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api) - -### Search Google Product -<%= snippet('python', 'tests/example_search_google_product.py', 9, 25) %> -see: [https://serpapi.com/google-product-api](https://serpapi.com/google-product-api) - -### Search Google Reverse Image -<%= snippet('python', 'tests/example_search_google_reverse_image.py', 9, 25) %> -see: [https://serpapi.com/google-reverse-image](https://serpapi.com/google-reverse-image) +Optionally, the HTTP connection can be tuned: + - timeout : connection timeout by default 60s + - retries : attempt to reconnect if the connection failed by default: False. Documentation: https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html#urllib3.util.Retry + - api_key : the secret user API available from http://serpapi.com/manage-api-key -### Search Google Events -<%= snippet('python', 'tests/example_search_google_events.py', 9, 25) %> -see: [https://serpapi.com/google-events-api](https://serpapi.com/google-events-api) + SerpApi's SLA is 99.95% successful searches: https://serpapi.com/faq/general/do-you-provide-sla-guarantees -### Search Google Local Services -<%= snippet('python', 'tests/example_search_google_local_services.py', 9, 25) %> -see: [https://serpapi.com/google-local-services-api](https://serpapi.com/google-local-services-api) +```python +client = serpapi.Client( + retries = 5, # or urllib3.util.Retry(connect=5, read=2) + timeout = 4.2, + api_key = "secret_api_key", # from serpapi.com +) +``` -### Search Google Maps -<%= snippet('python', 'tests/example_search_google_maps.py', 9, 25) %> -see: [https://serpapi.com/google-maps-api](https://serpapi.com/google-maps-api) +For more details: [`urllib3` documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html) -### Search Google Jobs -<%= snippet('python', 'tests/example_search_google_jobs.py', 9, 25) %> -see: [https://serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api) +## Basic example per search engines -### Search Google Play -<%= snippet('python', 'tests/example_search_google_play.py', 9, 25) %> -see: [https://serpapi.com/google-play-api](https://serpapi.com/google-play-api) +<%- Dir.glob('tests/example_search_*.py').each do |example| -%> +<% engine = example.match('example_search_(\w+)\.py')[1] %> -### Search Google Images -<%= snippet('python', 'tests/example_search_google_images.py', 9, 25) %> -see: [https://serpapi.com/images-results](https://serpapi.com/images-results) +### Search <%= engine.split('_').map(&:capitalize).join(' ') %> +<%= snippet('python', "tests/example_search_#{engine}.py", 9, 40) %> +
+Documentation: https://serpapi.com/<%= engine.tr('_', '-') %>-search-api +<%- end -%> # Developer Guide ### Key goals diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py index eb1d9b3..bc05673 100644 --- a/serpapi/serpapi.py +++ b/serpapi/serpapi.py @@ -1,39 +1,38 @@ """SerpApi client library for python""" import json import urllib3 +from typing import Optional, Union from .error import SerpApiException from .object_decoder import ObjectDecoder + class HttpClient: """Simple HTTP client wrapper around urllib3""" - BACKEND = 'https://serpapi.com' - SUPPORTED_DECODER = ['json', 'html', 'object'] + BACKEND = "https://serpapi.com" + SUPPORTED_DECODER = ["json", "html", "object"] - def __init__(self, parameter: dict = None): + def __init__( + self, + timeout: Optional[Union[Timeout, float, int]] = 60, + retries: Optional[Union[Retry, bool, int]] = False, + api_key: str = None, + ): # initialize the http client self.http = urllib3.PoolManager() - # initialize parameter - if parameter is None: - parameter = {} - self.parameter = parameter + self.parameter = {} # urllib3 configurations # HTTP connect timeout - if 'timeout' in parameter: - self.timeout = parameter['timeout'] - else: - # 60s default - self.timeout = 60.0 + self.timeout = timeout - # no HTTP retry - if 'retries' in parameter: - self.retries = parameter['retries'] - else: - self.retries = False + # No HTTP retries by default + self.retries = retries + + self.api_key = api_key - def start(self, path: str, parameter: dict = None, decoder: str = 'json'): + def start(self, path: str, parameter: dict = None, decoder: str = "json"): """start HTTP request and decode response using urllib3. The response is decoded using the selected decoder: - html: raw HTML response @@ -56,24 +55,26 @@ def start(self, path: str, parameter: dict = None, decoder: str = 'json'): dict|str|object decoded HTTP response""" # set client language - self.parameter['source'] = 'python' + self.parameter["source"] = "python" # set output type - if decoder == 'object': - self.parameter['output'] = 'json' + if decoder == "object": + self.parameter["output"] = "json" else: - self.parameter['output'] = decoder + self.parameter["output"] = decoder # merge parameter defaults and overrides fields = self.parameter.copy() fields.update(parameter) # execute HTTP get request - response = self.http.request('GET', - self.BACKEND + path, - fields=fields, - timeout=self.timeout, - retries=self.retries) + response = self.http.request( + "GET", + self.BACKEND + path, + fields=fields, + timeout=self.timeout, + retries=self.retries, + ) # decode response return self.decode(response, decoder) @@ -82,33 +83,34 @@ def decode(self, response: any, decoder: str): # handle HTTP error if response.status != 200: try: - raw = response.data.decode('utf-8') + raw = response.data.decode("utf-8") payload = json.loads(raw) - raise SerpApiException(payload['error']) + raise SerpApiException(payload["error"]) except Exception as ex: raise SerpApiException(raw) from ex # HTTP success 200 - payload = response.data.decode('utf-8') + payload = response.data.decode("utf-8") # successful response decoding - if decoder == 'json': + if decoder == "json": return json.loads(payload) - if decoder == 'html': + if decoder == "html": return payload - if decoder == 'object': + if decoder == "object": data = json.loads(payload) return ObjectDecoder(data).create() - raise SerpApiException("Invalid decoder: " + - decoder + ", available: json, html, object") + raise SerpApiException( + "Invalid decoder: " + decoder + ", available: json, html, object" + ) class Client(HttpClient): """ - Client performend http query to serpApi.com using urllib3 under the hood. + Client performed http query to SerpApi.com using urllib3 under the hood. The HTTP connection be tuned to allow - retries : attempt to reconnect if the connection fail by default: False @@ -118,13 +120,10 @@ class Client(HttpClient): """ def __init__(self, parameter: dict = None): - # define default parameter - if parameter is None: - parameter = {} # initialize HTTP client HttpClient.__init__(self, parameter) - def search(self, parameter: dict = None, decoder: str = 'json'): + def search(self, parameter: dict = None, decoder: str = "json"): """ make search then decode the output decoder supported 'json', 'html', 'object' @@ -144,7 +143,7 @@ def search(self, parameter: dict = None, decoder: str = 'json'): str if decoder = 'html' object if decoder = 'object' """ - return self.start(path='/search', parameter=parameter, decoder=decoder) + return self.start(path="/search", parameter=parameter, decoder=decoder) def html(self, parameter: dict = None): """ @@ -160,7 +159,7 @@ def html(self, parameter: dict = None): str raw html search results directly from the search engine """ - return self.start('/search', parameter, 'html') + return self.start("/search", parameter, "html") def location(self, parameter: dict = None): """ @@ -177,9 +176,9 @@ def location(self, parameter: dict = None): array list of matching locations """ - return self.start('/locations.json', parameter, 'json') + return self.start("/locations.json", parameter, "json") - def search_archive(self, search_id: str, decoder: str = 'json'): + def search_archive(self, search_id: str, decoder: str = "json"): """ Retrieve search results from the Search Archive API @@ -196,7 +195,7 @@ def search_archive(self, search_id: str, decoder: str = 'json'): else: path += decoder else: - raise SerpApiException('Decoder must be json or html or object') + raise SerpApiException("Decoder must be json or html or object") return self.start(path, {}, decoder) def account(self, api_key: str = None): @@ -214,5 +213,5 @@ def account(self, api_key: str = None): user account information """ if api_key is not None: - self.parameter['api_key'] = api_key - return self.start('/account', self.parameter, 'json') + self.parameter["api_key"] = api_key + return self.start("/account", self.parameter, "json") diff --git a/tests/example_search_google_local_services.py b/tests/example_search_google_local_services.py index 121b4ed..d5eab56 100644 --- a/tests/example_search_google_local_services.py +++ b/tests/example_search_google_local_services.py @@ -2,20 +2,23 @@ import unittest import os import serpapi +import urllib3 class TestGoogleLocalServices(unittest.TestCase): @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") def test_search_google_local_services(self): client = serpapi.Client({ - 'engine': 'google_local_services', - 'api_key': os.getenv("API_KEY") - }) + 'api_key': os.getenv("API_KEY"), + 'timeout': 61.1, + 'retries': 2 + }) + data = client.search({ + 'engine': 'google_local_services', 'q': 'Electrician', 'place_id': 'ChIJOwg_06VPwokRYv534QaPC8g', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['local_ads']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com/manage-api-key