diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java index 11a671f7a..69d6e8325 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java @@ -456,6 +456,11 @@ private void onDownloadSuccess(Request request, Page page) { } } else { logger.info("page status code error, page {} , code: {}", request.getUrl(), page.getStatusCode()); + if (site.getCycleRetryTimes() != 0) { + sleep(site.getRetrySleepTime()); + // for cycle retry + doCycleRetry(request); + } } sleep(site.getSleepTime()); return; @@ -465,6 +470,7 @@ private void onDownloaderFail(Request request) { if (site.getCycleRetryTimes() == 0) { sleep(site.getSleepTime()); } else { + sleep(site.getRetrySleepTime()); // for cycle retry doCycleRetry(request); } @@ -477,11 +483,10 @@ private void doCycleRetry(Request request) { } else { int cycleTriedTimes = (Integer) cycleTriedTimesObject; cycleTriedTimes++; - if (cycleTriedTimes < site.getCycleRetryTimes()) { + if (site.getCycleRetryTimes() < 0 || cycleTriedTimes < site.getCycleRetryTimes()) { addRequest(SerializationUtils.clone(request).setPriority(0).putExtra(Request.CYCLE_TRIED_TIMES, cycleTriedTimes)); } } - sleep(site.getRetrySleepTime()); } protected void sleep(int time) {