From 80441068df47125d9ea0850f1d3b4d82b77b43e6 Mon Sep 17 00:00:00 2001
From: Robert Irelan <rirelan@gmail.com>
Date: Tue, 18 Jan 2022 13:22:34 -0800
Subject: [PATCH 1/5] Retry PUT 409 errors - delete and re-create event
Attempt to address #963
I would appreciate a code review here - there might be a more elegant way to address this problem, but it works for my (very limited) use case.
---
vdirsyncer/storage/dav.py | 22 ++++++++++++++++++++--
1 file changed, 20 insertions(+), 2 deletions(-)
diff --git a/vdirsyncer/storage/dav.py b/vdirsyncer/storage/dav.py
index dcaf15e9..3f805294 100644
--- a/vdirsyncer/storage/dav.py
+++ b/vdirsyncer/storage/dav.py
@@ -593,12 +593,30 @@ async def _put(self, href, item, etag):
async def update(self, href, item, etag):
if etag is None:
raise ValueError("etag must be given and must not be None.")
- href, etag = await self._put(self._normalize_href(href), item, etag)
+ try:
+ href, etag = await self._put(self._normalize_href(href), item, etag)
+ except aiohttp.ClientResponseError as e:
+ if e.status == 409:
+ dav_logger.debug("Conflict, will delete old event and recreate it.")
+ await self.delete(self._normalize_href(href), None)
+ dav_logger.debug("Now trying again")
+ href, etag = await self._put(self._normalize_href(href), item, None)
+ else:
+ raise e
return etag
async def upload(self, item: Item):
href = self._get_href(item)
- rv = await self._put(href, item, None)
+ try:
+ rv = await self._put(href, item, None)
+ except aiohttp.ClientResponseError as e:
+ if e.status == 409:
+ dav_logger.debug("Conflict, will delete old event and recreate it.")
+ await self.delete(href, None)
+ dav_logger.debug("Now trying again")
+ rv = await self._put(href, item, None)
+ else:
+ raise e
return rv
async def delete(self, href, etag):
From 6139d6d3a6696bd56ec7e822b41444024fbac99b Mon Sep 17 00:00:00 2001
From: "robert.irelan" <robert.irelan@bytedance.com>
Date: Mon, 28 Mar 2022 13:24:41 -0700
Subject: [PATCH 2/5] Don't throw exceptions on unprocessed hrefs in get_multi
Rather, just log them. This allows my calendar sync to continue to work
even if some of the hrefs can't be retrieved. In my case, one of the
hrefs on my calendar was always returning a 404, so I want to skip it.
---
vdirsyncer/storage/dav.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/vdirsyncer/storage/dav.py b/vdirsyncer/storage/dav.py
index 3f805294..d4f5ab4e 100644
--- a/vdirsyncer/storage/dav.py
+++ b/vdirsyncer/storage/dav.py
@@ -553,7 +553,7 @@ async def get_multi(self, hrefs):
else:
rv.append((href, Item(raw), etag))
for href in hrefs_left:
- raise exceptions.NotFoundError(href)
+ dav_logger.warning(f"Skipping {href}, not found")
for href, item, etag in rv:
yield href, item, etag
@@ -651,6 +651,7 @@ def _parse_prop_responses(self, root, handled_hrefs=None):
props = response.findall("{DAV:}propstat/{DAV:}prop")
if props is None or not len(props):
dav_logger.debug(f"Skipping {href!r}, properties are missing.")
+ dav_logger.debug(f"Response for {href!r}: {etree.tostring(response)}")
continue
else:
props = _merge_xml(props)
From 3e5e4381c83902158d5190a7cdc7e6a80c869801 Mon Sep 17 00:00:00 2001
From: Robert Irelan <rirelan@gmail.com>
Date: Mon, 13 Mar 2023 12:46:50 -0700
Subject: [PATCH 3/5] More hacks to ignore errors from Google Calendar
---
vdirsyncer/http.py | 4 +++-
vdirsyncer/storage/dav.py | 38 +++++++++++++++++++++++++++++--------
vdirsyncer/sync/__init__.py | 1 -
3 files changed, 33 insertions(+), 10 deletions(-)
diff --git a/vdirsyncer/http.py b/vdirsyncer/http.py
index 30cfe909..bcf621cc 100644
--- a/vdirsyncer/http.py
+++ b/vdirsyncer/http.py
@@ -147,7 +147,9 @@ async def request(
logger.debug(response.status)
logger.debug(response.headers)
- logger.debug(response.content)
+ if (response.status >= 400 and hasattr(response, 'content')
+ and hasattr(response.content, '_buffer')):
+ logger.debug(response.content._buffer)
if response.status == 412:
raise exceptions.PreconditionFailed(response.reason)
diff --git a/vdirsyncer/storage/dav.py b/vdirsyncer/storage/dav.py
index d4f5ab4e..dad585e6 100644
--- a/vdirsyncer/storage/dav.py
+++ b/vdirsyncer/storage/dav.py
@@ -598,11 +598,22 @@ async def update(self, href, item, etag):
except aiohttp.ClientResponseError as e:
if e.status == 409:
dav_logger.debug("Conflict, will delete old event and recreate it.")
- await self.delete(self._normalize_href(href), None)
- dav_logger.debug("Now trying again")
- href, etag = await self._put(self._normalize_href(href), item, None)
+ try:
+ await self.delete(self._normalize_href(href), None)
+ dav_logger.debug("Now trying again")
+ rv = await self._put(self._normalize_href(href), item, None)
+ except aiohttp.ClientResponseError as delerr:
+ dav_logger.debug(f"delerr.status = {delerr.status}")
+ if delerr.status == 404:
+ dav_logger("Old event not found, ignoring")
+ rv = None, None
+ else:
+ raise
+ elif e.status == 403:
+ dav_logger.debug("Google Calendar refusing update, ignore")
+ rv = None, None
else:
- raise e
+ raise
return etag
async def upload(self, item: Item):
@@ -612,11 +623,22 @@ async def upload(self, item: Item):
except aiohttp.ClientResponseError as e:
if e.status == 409:
dav_logger.debug("Conflict, will delete old event and recreate it.")
- await self.delete(href, None)
- dav_logger.debug("Now trying again")
- rv = await self._put(href, item, None)
+ try:
+ await self.delete(href, None)
+ dav_logger.debug("Now trying again")
+ rv = await self._put(href, item, None)
+ except aiohttp.ClientResponseError as delerr:
+ dav_logger.debug(f"delerr.status = {delerr.status}")
+ if delerr.status == 404:
+ dav_logger.debug("Old event not found, ignoring")
+ rv = None, None
+ else:
+ raise
+ elif e.status == 403:
+ dav_logger.debug("Google Calendar refusing update, ignore")
+ rv = None, None
else:
- raise e
+ raise
return rv
async def delete(self, href, etag):
diff --git a/vdirsyncer/sync/__init__.py b/vdirsyncer/sync/__init__.py
index 8678bc9a..6e489dae 100644
--- a/vdirsyncer/sync/__init__.py
+++ b/vdirsyncer/sync/__init__.py
@@ -208,7 +208,6 @@ async def _run_impl(self, a, b):
)
)
href, etag = await self.dest.storage.upload(self.item)
- assert href is not None
self.dest.status.insert_ident(
self.ident, ItemMetadata(href=href, hash=self.item.hash, etag=etag)
From 966d711a17d3953fbe31c4744bd43452114b65f8 Mon Sep 17 00:00:00 2001
From: Robert Irelan <rirelan@gmail.com>
Date: Tue, 14 Mar 2023 17:23:55 -0700
Subject: [PATCH 4/5] Also ignore 404 errors during delete for conflicts
---
vdirsyncer/storage/dav.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/vdirsyncer/storage/dav.py b/vdirsyncer/storage/dav.py
index dad585e6..9726b10f 100644
--- a/vdirsyncer/storage/dav.py
+++ b/vdirsyncer/storage/dav.py
@@ -604,8 +604,8 @@ async def update(self, href, item, etag):
rv = await self._put(self._normalize_href(href), item, None)
except aiohttp.ClientResponseError as delerr:
dav_logger.debug(f"delerr.status = {delerr.status}")
- if delerr.status == 404:
- dav_logger("Old event not found, ignoring")
+ if delerr.status == 403 or delerr.status == 404:
+ dav_logger.warning("Old event not found, ignoring")
rv = None, None
else:
raise
@@ -629,8 +629,8 @@ async def upload(self, item: Item):
rv = await self._put(href, item, None)
except aiohttp.ClientResponseError as delerr:
dav_logger.debug(f"delerr.status = {delerr.status}")
- if delerr.status == 404:
- dav_logger.debug("Old event not found, ignoring")
+ if delerr.status == 403 or delerr.status == 404:
+ dav_logger.warning("Old event not found, ignoring")
rv = None, None
else:
raise
From b21e37e2b630b1c2a7da701d97936abd011cd20e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
<66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 15 Mar 2023 00:50:37 +0000
Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
---
vdirsyncer/http.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/vdirsyncer/http.py b/vdirsyncer/http.py
index bcf621cc..3a158862 100644
--- a/vdirsyncer/http.py
+++ b/vdirsyncer/http.py
@@ -147,8 +147,11 @@ async def request(
logger.debug(response.status)
logger.debug(response.headers)
- if (response.status >= 400 and hasattr(response, 'content')
- and hasattr(response.content, '_buffer')):
+ if (
+ response.status >= 400
+ and hasattr(response, "content")
+ and hasattr(response.content, "_buffer")
+ ):
logger.debug(response.content._buffer)
if response.status == 412: