diff --git a/Makefile b/Makefile
index 20b4823..e3a4d70 100644
--- a/Makefile
+++ b/Makefile
@@ -22,11 +22,11 @@ clean:
# lint check
lint:
- pylint serpapi
+ python3 -m pylint serpapi
# test with Python 3
test:
- pytest --cov=serpapi --cov-report html tests/*.py
+ python3 -m pytest --cov=serpapi --cov-report html tests/*.py
# install dependencies
#
@@ -58,12 +58,12 @@ oobt: build
check: oobt
- twine check ${dist}
+ python3 -m twine check ${dist}
release: # check
- twine upload ${dist}
+ python3 -m twine upload ${dist}
# run example only
# and display output (-s)
example:
- pytest -s "tests/test_example.py::TestExample::test_async"
+ python3 -m pytest -s "tests/test_example.py::TestExample::test_async"
diff --git a/README.md b/README.md
index bcba299..92a9d64 100644
--- a/README.md
+++ b/README.md
@@ -1,13 +1,13 @@
# User guide
[](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml)
-[SerpApi]](https://serpapi.com) allows to scrape any search engine results.
- It's easy, fast, easy, feature rich, cost effective, scalable and reliable.
+[SerpApi](https://serpapi.com) allows to scrape any search engine results.
+It's easy, fast, easy, feature rich, cost effective, scalable and reliable.
-This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi]](https://serpapi.com).
+This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi](https://serpapi.com).
This is an open source project hosted under https://github.com/serpapi/serpapi-python.
-SerpApi.com provides a [script builder]](https://serpapi.com/demo) to get you started quickly.
+SerpApi.com provides a [script builder](https://serpapi.com/demo) to get you started quickly.
## Installation
SerpApi can be installed with pip.
@@ -17,78 +17,73 @@ $ python -m pip install serpapi
```
## Quick start
-First things first, import the serpapi module:
+
+The following example runs a search for `"coffee"` using your secret API key which you can find at [SerpApi Dashboard](https://serpapi.com/manage-api-key) page. The `serpapi.search` object handles all of the details of connection pooling and thread safety so that you don't have to:
```python
import serpapi
-```
-You'll need a client instance to make a search. This object handles all of the details of connection pooling and thread safety so that you don't have to:
-```python
-client = serpapi.Client()
-```
-To make a search using SerpApi.com:
+client = serpapi.Client(
+ api_key = "secret_api_key", # from serpapi.com
+)
-```python
-parameter = {
- api_key: "secret_api_key", # from serpapi.com
- engine: "google", # search engine
- q: "coffee", # search topic
- location: "Austin,TX" # location
+parameters = {
+ "q": "coffee",
}
-results = searpapi.search(parameter)
-```
-Putting everything together.
-```python
-import serpapi
-parameter = {
- api_key: "secret_api_key", # from serpapi.com
- engine: "google", # search engine
- q: "coffee", # search topic
- location: "Austin,TX" # location
-}
-results = searpapi.search(parameter)
+results = client.search(parameters)
+
print(results)
```
### Advanced settings
SerpApi Client uses urllib3 under the hood.
-Optionally, rhe HTTP connection can be tuned:
+
+Optionally, the HTTP connection can be tuned:
- timeout : connection timeout by default 60s
- - retries : attempt to reconnect if the connection failed by default: False.
- serpapi is reliable at 99.99% but your company network might not be as stable.
-
- ```python
-parameter = {
- retries: 5,
- timeout: 4.0,
- # extra user parameters
-}
+ - retries : attempt to reconnect if the connection failed by default: False. Documentation: https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html#urllib3.util.Retry
+ - api_key : the secret user API available from http://serpapi.com/manage-api-key
+
+ SerpApi's SLA is 99.95% successful searches: https://serpapi.com/faq/general/do-you-provide-sla-guarantees
+
+```python
+client = serpapi.Client(
+ retries = 5, # or urllib3.util.Retry(connect=5, read=2)
+ timeout = 4.2,
+ api_key = "secret_api_key", # from serpapi.com
+)
```
-for more details: [URL LIB3 documentation]](https://urllib3.readthedocs.io/en/stable/user-guide.html)
+For more details: [`urllib3` documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html)
## Basic example per search engines
-### Search Bing
+
+
+
+### Search Google Local Services
```python
import serpapi
import pprint
import os
+ def test_search_google_local_services(self):
client = serpapi.Client({
-'engine': 'bing',
-'api_key': os.getenv("API_KEY")
+ 'api_key': os.getenv("API_KEY"),
+ 'timeout': 61.1,
+ 'retries': 2
})
data = client.search({
-'q': 'coffee',
+ 'engine': 'google_local_services',
+ 'q': 'Electrician',
+ 'place_id': 'ChIJOwg_06VPwokRYv534QaPC8g',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['organic_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['local_ads'])
```
- test: tests/example_search_bing.py
-see: [https://serpapi.com/bing-search-api](https://serpapi.com/bing-search-api)
+test: tests/example_search_google_local_services.py
+
+Documentation: https://serpapi.com/google-local-services-search-api
+
### Search Baidu
```python
@@ -97,407 +92,425 @@ import pprint
import os
client = serpapi.Client({
-'engine': 'baidu',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'baidu',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'q': 'coffee',
+ 'q': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
pp.pprint(data['organic_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
```
- test: tests/example_search_baidu.py
-see: [https://serpapi.com/baidu-search-api](https://serpapi.com/baidu-search-api)
+test: tests/example_search_baidu.py
+
+Documentation: https://serpapi.com/baidu-search-api
-### Search Yahoo
+
+### Search Ebay
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'yahoo',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'ebay',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'p': 'coffee',
+ '_nkw': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
pp.pprint(data['organic_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
```
- test: tests/example_search_yahoo.py
-see: [https://serpapi.com/yahoo-search-api](https://serpapi.com/yahoo-search-api)
+test: tests/example_search_ebay.py
+
+Documentation: https://serpapi.com/ebay-search-api
-### Search Youtube
+
+### Search Apple App Store
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'youtube',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'apple_app_store',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'search_query': 'coffee',
+ 'term': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['video_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['organic_results'])
```
- test: tests/example_search_youtube.py
-see: [https://serpapi.com/youtube-search-api](https://serpapi.com/youtube-search-api)
+test: tests/example_search_apple_app_store.py
+
+Documentation: https://serpapi.com/apple-app-store-search-api
-### Search Walmart
+
+### Search Google Product
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'walmart',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'google_product',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'query': 'coffee',
+ 'q': 'coffee',
+ 'product_id': '4172129135583325756',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['organic_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['product_results'])
```
- test: tests/example_search_walmart.py
-see: [https://serpapi.com/walmart-search-api](https://serpapi.com/walmart-search-api)
+test: tests/example_search_google_product.py
+
+Documentation: https://serpapi.com/google-product-search-api
-### Search Ebay
+
+### Search Google Maps
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'ebay',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'google_maps',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'_nkw': 'coffee',
+ 'q': 'pizza',
+ 'll': '@40.7455096,-74.0083012,15.1z',
+ 'type': 'search',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['organic_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['local_results'])
```
- test: tests/example_search_ebay.py
-see: [https://serpapi.com/ebay-search-api](https://serpapi.com/ebay-search-api)
+test: tests/example_search_google_maps.py
+
+Documentation: https://serpapi.com/google-maps-search-api
-### Search Naver
+
+### Search Google Images
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'naver',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'google',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'query': 'coffee',
+ 'engine': 'google',
+ 'tbm': 'isch',
+ 'q': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['ads_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['images_results'])
```
- test: tests/example_search_naver.py
-see: [https://serpapi.com/naver-search-api](https://serpapi.com/naver-search-api)
+test: tests/example_search_google_images.py
+
+Documentation: https://serpapi.com/google-images-search-api
-### Search Home Depot
+
+### Search Google Search
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'home_depot',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'google',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'q': 'table',
+ 'q': 'coffee',
+ 'engine': 'google',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['products'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['organic_results'])
```
- test: tests/example_search_home_depot.py
-see: [https://serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api)
+test: tests/example_search_google_search.py
+
+Documentation: https://serpapi.com/google-search-search-api
-### Search Apple App Store
+
+### Search Google Jobs
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'apple_app_store',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'google_jobs',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'term': 'coffee',
+ 'q': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['organic_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['jobs_results'])
```
- test: tests/example_search_apple_app_store.py
-see: [https://serpapi.com/apple-app-store](https://serpapi.com/apple-app-store)
+test: tests/example_search_google_jobs.py
+
+Documentation: https://serpapi.com/google-jobs-search-api
-### Search Duckduckgo
+
+### Search Naver
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'duckduckgo',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'naver',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'q': 'coffee',
+ 'query': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['organic_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['ads_results'])
```
- test: tests/example_search_duckduckgo.py
-see: [https://serpapi.com/duckduckgo-search-api](https://serpapi.com/duckduckgo-search-api)
+test: tests/example_search_naver.py
+
+Documentation: https://serpapi.com/naver-search-api
-### Search Google Search
+
+### Search Google Scholar
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'google',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'google_scholar',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'q': 'coffee',
-'engine': 'google',
+ 'q': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
pp.pprint(data['organic_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
```
- test: tests/example_search_google_search.py
-see: [https://serpapi.com/search-api](https://serpapi.com/search-api)
+test: tests/example_search_google_scholar.py
+
+Documentation: https://serpapi.com/google-scholar-search-api
-### Search Google Scholar
+
+### Search Youtube
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'google_scholar',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'youtube',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'q': 'coffee',
+ 'search_query': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['organic_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['video_results'])
```
- test: tests/example_search_google_scholar.py
-see: [https://serpapi.com/google-scholar-api](https://serpapi.com/google-scholar-api)
+test: tests/example_search_youtube.py
+
+Documentation: https://serpapi.com/youtube-search-api
-### Search Google Autocomplete
+
+### Search Google Reverse Image
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'google_autocomplete',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'google_reverse_image',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'q': 'coffee',
+ 'image_url': 'https://i.imgur.com/5bGzZi7.jpg',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['suggestions'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['image_sizes'])
```
- test: tests/example_search_google_autocomplete.py
-see: [https://serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api)
+test: tests/example_search_google_reverse_image.py
+
+Documentation: https://serpapi.com/google-reverse-image-search-api
-### Search Google Product
+
+### Search Yahoo
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'google_product',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'yahoo',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'q': 'coffee',
-'product_id': '4172129135583325756',
+ 'p': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['product_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['organic_results'])
```
- test: tests/example_search_google_product.py
-see: [https://serpapi.com/google-product-api](https://serpapi.com/google-product-api)
+test: tests/example_search_yahoo.py
+
+Documentation: https://serpapi.com/yahoo-search-api
-### Search Google Reverse Image
+
+### Search Duckduckgo
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'google_reverse_image',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'duckduckgo',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'image_url': 'https://i.imgur.com/5bGzZi7.jpg',
+ 'q': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['image_sizes'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['organic_results'])
```
- test: tests/example_search_google_reverse_image.py
-see: [https://serpapi.com/google-reverse-image](https://serpapi.com/google-reverse-image)
+test: tests/example_search_duckduckgo.py
+
+Documentation: https://serpapi.com/duckduckgo-search-api
-### Search Google Events
+
+### Search Bing
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'google_events',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'bing',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'q': 'coffee',
+ 'q': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['events_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['organic_results'])
```
- test: tests/example_search_google_events.py
-see: [https://serpapi.com/google-events-api](https://serpapi.com/google-events-api)
+test: tests/example_search_bing.py
+
+Documentation: https://serpapi.com/bing-search-api
-### Search Google Local Services
+
+### Search Google Autocomplete
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'google_local_services',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'google_autocomplete',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'q': 'Electrician',
-'place_id': 'ChIJOwg_06VPwokRYv534QaPC8g',
+ 'q': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['local_ads'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['suggestions'])
```
- test: tests/example_search_google_local_services.py
-see: [https://serpapi.com/google-local-services-api](https://serpapi.com/google-local-services-api)
+test: tests/example_search_google_autocomplete.py
+
+Documentation: https://serpapi.com/google-autocomplete-search-api
-### Search Google Maps
+
+### Search Home Depot
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'google_maps',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'home_depot',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'q': 'pizza',
-'ll': '@40.7455096,-74.0083012,15.1z',
-'type': 'search',
+ 'q': 'table',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['local_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['products'])
```
- test: tests/example_search_google_maps.py
-see: [https://serpapi.com/google-maps-api](https://serpapi.com/google-maps-api)
+test: tests/example_search_home_depot.py
+
+Documentation: https://serpapi.com/home-depot-search-api
-### Search Google Jobs
+
+### Search Walmart
```python
import serpapi
import pprint
import os
client = serpapi.Client({
-'engine': 'google_jobs',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'walmart',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'q': 'coffee',
+ 'query': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['jobs_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['organic_results'])
```
- test: tests/example_search_google_jobs.py
-see: [https://serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api)
+test: tests/example_search_walmart.py
+
+Documentation: https://serpapi.com/walmart-search-api
-### Search Google Play
+
+### Search Google Events
```python
import serpapi
import pprint
import os
-def test_search_google_play(self):
client = serpapi.Client({
-'engine': 'google_play',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'google_events',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'q': 'kite',
-'store': 'apps',
+ 'q': 'coffee',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['organic_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['events_results'])
```
- test: tests/example_search_google_play.py
-see: [https://serpapi.com/google-play-api](https://serpapi.com/google-play-api)
+test: tests/example_search_google_events.py
+
+Documentation: https://serpapi.com/google-events-search-api
-### Search Google Images
+
+### Search Google Play
```python
import serpapi
import pprint
import os
+ def test_search_google_play(self):
client = serpapi.Client({
-'engine': 'google',
-'api_key': os.getenv("API_KEY")
-})
+ 'engine': 'google_play',
+ 'api_key': os.getenv("API_KEY")
+ })
data = client.search({
-'engine': 'google',
-'tbm': 'isch',
-'q': 'coffee',
+ 'q': 'kite',
+ 'store': 'apps',
})
pp = pprint.PrettyPrinter(indent=2)
-pp.pprint(data['images_results'])
-# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
+pp.pprint(data['organic_results'])
```
- test: tests/example_search_google_images.py
-see: [https://serpapi.com/images-results](https://serpapi.com/images-results)
+test: tests/example_search_google_play.py
+
+Documentation: https://serpapi.com/google-play-search-api
# Developer Guide
### Key goals
diff --git a/README.md.erb b/README.md.erb
index 9e1fb9f..2f6a9ac 100644
--- a/README.md.erb
+++ b/README.md.erb
@@ -1,22 +1,22 @@
<%-
def snippet(format, path, start, stop)
slice = File.new(path).readlines[start..stop]
- slice.reject! { |l| l =~ /self.assertIsNone\(/ }
- buf = slice.map { |l| l.gsub(/(^\s+)/, '')}.join
+ slice.reject! { |l| l.match?(/self.assertIsNone\(/) || l.match?(/# os\.getenv\("API_KEY"\)/) }
+ buf = slice.map { |l| l.gsub(/(^\s{4})/, '').gsub(/^\s*$/, '') }.join
buf.gsub!('self.assertIsNotNone(', "pp = pprint.PrettyPrinter(indent=2)\npp.pprint(")
- %Q(```#{format}\nimport serpapi\nimport pprint\nimport os\n\n#{buf}```\n test: #{path})
+ %Q(```#{format}\nimport serpapi\nimport pprint\nimport os\n\n#{buf}```\ntest: #{path})
end
-%>
# User guide
[](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml)
-[SerpApi]](https://serpapi.com) allows to scrape any search engine results.
- It's easy, fast, easy, feature rich, cost effective, scalable and reliable.
+[SerpApi](https://serpapi.com) allows to scrape any search engine results.
+It's easy, fast, easy, feature rich, cost effective, scalable and reliable.
-This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi]](https://serpapi.com).
+This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi](https://serpapi.com).
This is an open source project hosted under https://github.com/serpapi/serpapi-python.
-SerpApi.com provides a [script builder]](https://serpapi.com/demo) to get you started quickly.
+SerpApi.com provides a [script builder](https://serpapi.com/demo) to get you started quickly.
## Installation
SerpApi can be installed with pip.
@@ -26,142 +26,55 @@ $ python -m pip install serpapi
```
## Quick start
-First things first, import the serpapi module:
+
+The following example runs a search for `"coffee"` using your secret API key which you can find at [SerpApi Dashboard](https://serpapi.com/manage-api-key) page. The `serpapi.search` object handles all of the details of connection pooling and thread safety so that you don't have to:
```python
import serpapi
-```
-You'll need a client instance to make a search. This object handles all of the details of connection pooling and thread safety so that you don't have to:
-```python
-client = serpapi.Client()
-```
-To make a search using SerpApi.com:
+client = serpapi.Client(
+ api_key = "secret_api_key", # from serpapi.com
+)
-```python
-parameter = {
- api_key: "secret_api_key", # from serpapi.com
- engine: "google", # search engine
- q: "coffee", # search topic
- location: "Austin,TX" # location
+parameters = {
+ "q": "coffee",
}
-results = searpapi.search(parameter)
-```
-Putting everything together.
-```python
-import serpapi
-parameter = {
- api_key: "secret_api_key", # from serpapi.com
- engine: "google", # search engine
- q: "coffee", # search topic
- location: "Austin,TX" # location
-}
-results = searpapi.search(parameter)
+results = client.search(parameters)
+
print(results)
```
### Advanced settings
SerpApi Client uses urllib3 under the hood.
-Optionally, rhe HTTP connection can be tuned:
- - timeout : connection timeout by default 60s
- - retries : attempt to reconnect if the connection failed by default: False.
- serpapi is reliable at 99.99% but your company network might not be as stable.
-
- ```python
-parameter = {
- retries: 5,
- timeout: 4.0,
- # extra user parameters
-}
-```
-
-for more details: [URL LIB3 documentation]](https://urllib3.readthedocs.io/en/stable/user-guide.html)
-
-## Basic example per search engines
-### Search Bing
-<%= snippet('python', 'tests/example_search_bing.py', 9, 25) %>
-see: [https://serpapi.com/bing-search-api](https://serpapi.com/bing-search-api)
-
-### Search Baidu
-<%= snippet('python', 'tests/example_search_baidu.py', 9, 25) %>
-see: [https://serpapi.com/baidu-search-api](https://serpapi.com/baidu-search-api)
-### Search Yahoo
-<%= snippet('python', 'tests/example_search_yahoo.py', 9, 25) %>
-see: [https://serpapi.com/yahoo-search-api](https://serpapi.com/yahoo-search-api)
-
-### Search Youtube
-<%= snippet('python', 'tests/example_search_youtube.py', 9, 25) %>
-see: [https://serpapi.com/youtube-search-api](https://serpapi.com/youtube-search-api)
-
-### Search Walmart
-<%= snippet('python', 'tests/example_search_walmart.py', 9, 25) %>
-see: [https://serpapi.com/walmart-search-api](https://serpapi.com/walmart-search-api)
-
-### Search Ebay
-<%= snippet('python', 'tests/example_search_ebay.py', 9, 25) %>
-see: [https://serpapi.com/ebay-search-api](https://serpapi.com/ebay-search-api)
-
-### Search Naver
-<%= snippet('python', 'tests/example_search_naver.py', 9, 25) %>
-see: [https://serpapi.com/naver-search-api](https://serpapi.com/naver-search-api)
-
-### Search Home Depot
-<%= snippet('python', 'tests/example_search_home_depot.py', 9, 25) %>
-see: [https://serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api)
-
-### Search Apple App Store
-<%= snippet('python', 'tests/example_search_apple_app_store.py', 9, 25) %>
-see: [https://serpapi.com/apple-app-store](https://serpapi.com/apple-app-store)
-
-### Search Duckduckgo
-<%= snippet('python', 'tests/example_search_duckduckgo.py', 9, 25) %>
-see: [https://serpapi.com/duckduckgo-search-api](https://serpapi.com/duckduckgo-search-api)
-
-### Search Google Search
-<%= snippet('python', 'tests/example_search_google_search.py', 9, 25) %>
-see: [https://serpapi.com/search-api](https://serpapi.com/search-api)
-
-### Search Google Scholar
-<%= snippet('python', 'tests/example_search_google_scholar.py', 9, 25) %>
-see: [https://serpapi.com/google-scholar-api](https://serpapi.com/google-scholar-api)
-
-### Search Google Autocomplete
-<%= snippet('python', 'tests/example_search_google_autocomplete.py', 9, 25) %>
-see: [https://serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api)
-
-### Search Google Product
-<%= snippet('python', 'tests/example_search_google_product.py', 9, 25) %>
-see: [https://serpapi.com/google-product-api](https://serpapi.com/google-product-api)
-
-### Search Google Reverse Image
-<%= snippet('python', 'tests/example_search_google_reverse_image.py', 9, 25) %>
-see: [https://serpapi.com/google-reverse-image](https://serpapi.com/google-reverse-image)
+Optionally, the HTTP connection can be tuned:
+ - timeout : connection timeout by default 60s
+ - retries : attempt to reconnect if the connection failed by default: False. Documentation: https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html#urllib3.util.Retry
+ - api_key : the secret user API available from http://serpapi.com/manage-api-key
-### Search Google Events
-<%= snippet('python', 'tests/example_search_google_events.py', 9, 25) %>
-see: [https://serpapi.com/google-events-api](https://serpapi.com/google-events-api)
+ SerpApi's SLA is 99.95% successful searches: https://serpapi.com/faq/general/do-you-provide-sla-guarantees
-### Search Google Local Services
-<%= snippet('python', 'tests/example_search_google_local_services.py', 9, 25) %>
-see: [https://serpapi.com/google-local-services-api](https://serpapi.com/google-local-services-api)
+```python
+client = serpapi.Client(
+ retries = 5, # or urllib3.util.Retry(connect=5, read=2)
+ timeout = 4.2,
+ api_key = "secret_api_key", # from serpapi.com
+)
+```
-### Search Google Maps
-<%= snippet('python', 'tests/example_search_google_maps.py', 9, 25) %>
-see: [https://serpapi.com/google-maps-api](https://serpapi.com/google-maps-api)
+For more details: [`urllib3` documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html)
-### Search Google Jobs
-<%= snippet('python', 'tests/example_search_google_jobs.py', 9, 25) %>
-see: [https://serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api)
+## Basic example per search engines
-### Search Google Play
-<%= snippet('python', 'tests/example_search_google_play.py', 9, 25) %>
-see: [https://serpapi.com/google-play-api](https://serpapi.com/google-play-api)
+<%- Dir.glob('tests/example_search_*.py').each do |example| -%>
+<% engine = example.match('example_search_(\w+)\.py')[1] %>
-### Search Google Images
-<%= snippet('python', 'tests/example_search_google_images.py', 9, 25) %>
-see: [https://serpapi.com/images-results](https://serpapi.com/images-results)
+### Search <%= engine.split('_').map(&:capitalize).join(' ') %>
+<%= snippet('python', "tests/example_search_#{engine}.py", 9, 40) %>
+
+Documentation: https://serpapi.com/<%= engine.tr('_', '-') %>-search-api
+<%- end -%>
# Developer Guide
### Key goals
diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py
index eb1d9b3..bc05673 100644
--- a/serpapi/serpapi.py
+++ b/serpapi/serpapi.py
@@ -1,39 +1,38 @@
"""SerpApi client library for python"""
import json
import urllib3
+from typing import Optional, Union
from .error import SerpApiException
from .object_decoder import ObjectDecoder
+
class HttpClient:
"""Simple HTTP client wrapper around urllib3"""
- BACKEND = 'https://serpapi.com'
- SUPPORTED_DECODER = ['json', 'html', 'object']
+ BACKEND = "https://serpapi.com"
+ SUPPORTED_DECODER = ["json", "html", "object"]
- def __init__(self, parameter: dict = None):
+ def __init__(
+ self,
+ timeout: Optional[Union[Timeout, float, int]] = 60,
+ retries: Optional[Union[Retry, bool, int]] = False,
+ api_key: str = None,
+ ):
# initialize the http client
self.http = urllib3.PoolManager()
- # initialize parameter
- if parameter is None:
- parameter = {}
- self.parameter = parameter
+ self.parameter = {}
# urllib3 configurations
# HTTP connect timeout
- if 'timeout' in parameter:
- self.timeout = parameter['timeout']
- else:
- # 60s default
- self.timeout = 60.0
+ self.timeout = timeout
- # no HTTP retry
- if 'retries' in parameter:
- self.retries = parameter['retries']
- else:
- self.retries = False
+ # No HTTP retries by default
+ self.retries = retries
+
+ self.api_key = api_key
- def start(self, path: str, parameter: dict = None, decoder: str = 'json'):
+ def start(self, path: str, parameter: dict = None, decoder: str = "json"):
"""start HTTP request and decode response using urllib3.
The response is decoded using the selected decoder:
- html: raw HTML response
@@ -56,24 +55,26 @@ def start(self, path: str, parameter: dict = None, decoder: str = 'json'):
dict|str|object
decoded HTTP response"""
# set client language
- self.parameter['source'] = 'python'
+ self.parameter["source"] = "python"
# set output type
- if decoder == 'object':
- self.parameter['output'] = 'json'
+ if decoder == "object":
+ self.parameter["output"] = "json"
else:
- self.parameter['output'] = decoder
+ self.parameter["output"] = decoder
# merge parameter defaults and overrides
fields = self.parameter.copy()
fields.update(parameter)
# execute HTTP get request
- response = self.http.request('GET',
- self.BACKEND + path,
- fields=fields,
- timeout=self.timeout,
- retries=self.retries)
+ response = self.http.request(
+ "GET",
+ self.BACKEND + path,
+ fields=fields,
+ timeout=self.timeout,
+ retries=self.retries,
+ )
# decode response
return self.decode(response, decoder)
@@ -82,33 +83,34 @@ def decode(self, response: any, decoder: str):
# handle HTTP error
if response.status != 200:
try:
- raw = response.data.decode('utf-8')
+ raw = response.data.decode("utf-8")
payload = json.loads(raw)
- raise SerpApiException(payload['error'])
+ raise SerpApiException(payload["error"])
except Exception as ex:
raise SerpApiException(raw) from ex
# HTTP success 200
- payload = response.data.decode('utf-8')
+ payload = response.data.decode("utf-8")
# successful response decoding
- if decoder == 'json':
+ if decoder == "json":
return json.loads(payload)
- if decoder == 'html':
+ if decoder == "html":
return payload
- if decoder == 'object':
+ if decoder == "object":
data = json.loads(payload)
return ObjectDecoder(data).create()
- raise SerpApiException("Invalid decoder: " +
- decoder + ", available: json, html, object")
+ raise SerpApiException(
+ "Invalid decoder: " + decoder + ", available: json, html, object"
+ )
class Client(HttpClient):
"""
- Client performend http query to serpApi.com using urllib3 under the hood.
+ Client performed http query to SerpApi.com using urllib3 under the hood.
The HTTP connection be tuned to allow
- retries : attempt to reconnect if the connection fail by default: False
@@ -118,13 +120,10 @@ class Client(HttpClient):
"""
def __init__(self, parameter: dict = None):
- # define default parameter
- if parameter is None:
- parameter = {}
# initialize HTTP client
HttpClient.__init__(self, parameter)
- def search(self, parameter: dict = None, decoder: str = 'json'):
+ def search(self, parameter: dict = None, decoder: str = "json"):
"""
make search then decode the output
decoder supported 'json', 'html', 'object'
@@ -144,7 +143,7 @@ def search(self, parameter: dict = None, decoder: str = 'json'):
str if decoder = 'html'
object if decoder = 'object'
"""
- return self.start(path='/search', parameter=parameter, decoder=decoder)
+ return self.start(path="/search", parameter=parameter, decoder=decoder)
def html(self, parameter: dict = None):
"""
@@ -160,7 +159,7 @@ def html(self, parameter: dict = None):
str
raw html search results directly from the search engine
"""
- return self.start('/search', parameter, 'html')
+ return self.start("/search", parameter, "html")
def location(self, parameter: dict = None):
"""
@@ -177,9 +176,9 @@ def location(self, parameter: dict = None):
array
list of matching locations
"""
- return self.start('/locations.json', parameter, 'json')
+ return self.start("/locations.json", parameter, "json")
- def search_archive(self, search_id: str, decoder: str = 'json'):
+ def search_archive(self, search_id: str, decoder: str = "json"):
"""
Retrieve search results from the Search Archive API
@@ -196,7 +195,7 @@ def search_archive(self, search_id: str, decoder: str = 'json'):
else:
path += decoder
else:
- raise SerpApiException('Decoder must be json or html or object')
+ raise SerpApiException("Decoder must be json or html or object")
return self.start(path, {}, decoder)
def account(self, api_key: str = None):
@@ -214,5 +213,5 @@ def account(self, api_key: str = None):
user account information
"""
if api_key is not None:
- self.parameter['api_key'] = api_key
- return self.start('/account', self.parameter, 'json')
+ self.parameter["api_key"] = api_key
+ return self.start("/account", self.parameter, "json")
diff --git a/tests/example_search_google_local_services.py b/tests/example_search_google_local_services.py
index 121b4ed..d5eab56 100644
--- a/tests/example_search_google_local_services.py
+++ b/tests/example_search_google_local_services.py
@@ -2,20 +2,23 @@
import unittest
import os
import serpapi
+import urllib3
class TestGoogleLocalServices(unittest.TestCase):
@unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided")
def test_search_google_local_services(self):
client = serpapi.Client({
- 'engine': 'google_local_services',
- 'api_key': os.getenv("API_KEY")
- })
+ 'api_key': os.getenv("API_KEY"),
+ 'timeout': 61.1,
+ 'retries': 2
+ })
+
data = client.search({
+ 'engine': 'google_local_services',
'q': 'Electrician',
'place_id': 'ChIJOwg_06VPwokRYv534QaPC8g',
})
self.assertIsNone(data.get('error'))
self.assertIsNotNone(data['local_ads'])
- # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com
-
\ No newline at end of file
+ # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com/manage-api-key