Init

Author: elacuesta

.gitignore

@@ -0,0 +1,13 @@
+*.pyc
+.~lock*
+.DS_Store
+.mypy_cache/
+*.egg-info/
+.tox/
+.coverage
+htmlcov/
+coverage.xml
+build/
+dist/
+examples/
+pip-wheel-metadata/

LICENSE

@@ -0,0 +1,26 @@
+Copyright 2020 Eugenio Lacuesta
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Makefile

@@ -0,0 +1,14 @@
+.PHONY: lint types black clean
+
+lint:
+	@python -m flake8 --exclude=.git,venv* scrapy_playwright/*.py tests/*.py
+
+types:
+	@mypy --ignore-missing-imports --follow-imports=skip scrapy_playwright/*.py tests/*.py
+
+black:
+	@black --check scrapy_playwright tests
+
+clean:
+	@find . -name "*.pyc" -delete
+	@rm -rf .mypy_cache/ .tox/ build/ dist/ htmlcov/ .coverage coverage.xml

pyproject.toml

@@ -0,0 +1,2 @@
+[tool.black]
+line-length = 99

requirements-dev.txt

@@ -0,0 +1,4 @@
+black>=19.10b0
+flake8>=3.7.9
+mypy==0.780
+tox>=3.14

requirements.txt

@@ -0,0 +1,2 @@
+playwright>=0.7.0
+scrapy>=2.0

scrapy_playwright/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

scrapy_playwright/handler.py

@@ -0,0 +1,77 @@
+import logging
+from time import time
+from typing import Type, TypeVar
+
+from playwright import AsyncPlaywrightContextManager
+from scrapy import Spider, signals
+from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
+from scrapy.crawler import Crawler
+from scrapy.http import Request, Response
+from scrapy.http.headers import Headers
+from scrapy.responsetypes import responsetypes
+from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.reactor import verify_installed_reactor
+from twisted.internet.defer import Deferred, inlineCallbacks
+
+
+logger = logging.getLogger("scrapy-playwright")
+PlaywrightHandler = TypeVar("PlaywrightHandler", bound="ScrapyPlaywrightDownloadHandler")
+
+
+class ScrapyPlaywrightDownloadHandler(HTTPDownloadHandler):
+    def __init__(self, crawler: Crawler) -> None:
+        super().__init__(settings=crawler.settings, crawler=crawler)
+        verify_installed_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+        crawler.signals.connect(self._engine_started, signals.engine_started)
+        self.stats = crawler.stats
+
+    @classmethod
+    def from_crawler(cls: Type[PlaywrightHandler], crawler: Crawler) -> PlaywrightHandler:
+        return cls(crawler)
+
+    def _engine_started(self) -> Deferred:
+        return deferred_from_coro(self._launch_browser())
+
+    async def _launch_browser(self) -> None:
+        self.playwright_context_manager = AsyncPlaywrightContextManager()
+        self.playwright = await self.playwright_context_manager.start()
+        # FIXME: chromium hard-coded during initial development
+        self.browser = await self.playwright.chromium.launch()
+
+    @inlineCallbacks
+    def close(self) -> Deferred:
+        yield super().close()
+        if self.browser:
+            yield deferred_from_coro(self.browser.close())
+        yield deferred_from_coro(self.playwright_context_manager.__aexit__())
+
+    def download_request(self, request: Request, spider: Spider) -> Deferred:
+        if request.meta.get("playwright"):
+            return deferred_from_coro(self._download_request_playwright(request, spider))
+        return super().download_request(request, spider)
+
+    async def _download_request_playwright(self, request: Request, spider: Spider) -> Response:
+        page = await self.browser.newPage()  # type: ignore
+        self.stats.inc_value("playwright/page_count")
+
+        start_time = time()
+        response = await page.goto(request.url)
+
+        body = (await page.content()).encode("utf8")
+        request.meta["download_latency"] = time() - start_time
+
+        await page.screenshot(path="page.png")  # FIXME: only for development
+        await page.close()
+        self.stats.inc_value("playwright/page_count/closed")
+
+        headers = Headers(response.headers)
+        headers.pop("Content-Encoding", None)
+        respcls = responsetypes.from_args(headers=headers, url=page.url, body=body)
+        return respcls(
+            url=page.url,
+            status=response.status,
+            headers=headers,
+            body=body,
+            request=request,
+            flags=["playwright"],
+        )

setup.cfg

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 99

setup.py

@@ -0,0 +1,32 @@
+import setuptools
+
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+
+setuptools.setup(
+    name="scrapy-playwright",
+    version="0.0.1",
+    license="BSD",
+    description="Playwright integration for Scrapy",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    author="Eugenio Lacuesta",
+    author_email="eugenio.lacuesta@gmail.com",
+    url="https://github.com/elacuesta/scrapy-playwright",
+    packages=["scrapy_playwright"],
+    classifiers=[
+        "Development Status :: 1 - Planning",
+        "License :: OSI Approved :: BSD License",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Framework :: Scrapy",
+        "Intended Audience :: Developers",
+        "Topic :: Internet :: WWW/HTTP",
+        "Topic :: Software Development :: Libraries :: Application Frameworks",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+    ],
+    install_requires=["scrapy>=2.0", "playwright>=0.7.0"],
+)

tox.ini

@@ -0,0 +1,18 @@
+[tox]
+envlist = py36,py37,py38
+
+
+[testenv]
+deps =
+    -rrequirements.txt
+    -rtests/requirements.txt
+commands =
+    py.test --reactor=asyncio --cov=scrapy_playwright --cov-report=term-missing --cov-report=html --cov-report=xml {posargs: scrapy_playwright tests}
+
+
+[testenv:py37]
+basepython = python3.7
+
+
+[testenv:py38]
+basepython = python3.8