Skip to content

Add update time to URL prioritization rules #681

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add update time to URL prioritization rules #681

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
90c45ce
Add update time to URL prioritization rules
FedericoCeratto Jun 14, 2023
b03a534
Update tests
FedericoCeratto Jun 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions api/debian/changelog
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
ooni-api (1.0.59) unstable; urgency=medium

* Add update time to URL priorities

-- Federico Ceratto <federico@debian.org> Wed, 14 Jun 2023 18:20:00 +0100

ooni-api (1.0.58) unstable; urgency=medium

* Add incident management CRUD
Expand Down
3 changes: 2 additions & 1 deletion api/ooniapi/citizenlab.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
It contains rules on category_code, cc, domain and url to assign priorities.
Values can be wildcards "*". A citizenlab entry can match multiple rules.
"""
# The tables creation is in tests/integ/clickhouse_1_schema.sql

log = logging.getLogger() # overridden by current_app.logger

Expand Down Expand Up @@ -703,7 +704,7 @@ def list_url_priorities() -> Response:
global log
log = current_app.logger
log.debug("listing URL prio rules")
query = """SELECT category_code, cc, domain, url, priority
query = """SELECT category_code, cc, domain, url, priority, update_time
FROM url_priorities FINAL
ORDER BY category_code, cc, domain, url, priority
"""
Expand Down
3 changes: 2 additions & 1 deletion api/tests/integ/clickhouse_1_schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ CREATE TABLE default.url_priorities (
`cc` String,
`domain` String,
`url` String,
`priority` Int32
`priority` Int32,
`update_time` DateTime DEFAULT now()
)
ENGINE = CollapsingMergeTree(sign)
ORDER BY (category_code, cc, domain, url, priority)
Expand Down
12 changes: 8 additions & 4 deletions api/tests/integ/test_citizenlab.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,9 +360,13 @@ def match(url):
assert r.status_code == 200, r.json
for x in r.json["rules"]:
for k, v in x.items():
assert v != '', f"Empty value in {x}"
match = [x for x in r.json["rules"] if x == exp]
return len(match)
assert v != "", f"Empty value in {x}"
for ru in r.json["rules"]:
ru.pop("update_time")
if ru == exp:
return True # found

return False

assert match("INTEG-TEST") == 0
assert match("INTEG-TEST2") == 0
Expand All @@ -376,7 +380,7 @@ def match(url):
assert r.status_code == 400, r.json

# Create
xxx = dict(category_code="NEWS", priority=100, cc='', url="INTEG-TEST")
xxx = dict(category_code="NEWS", priority=100, cc="", url="INTEG-TEST")
d = dict(new_entry=xxx)
r = adminsession.post("/api/_/url-priorities/update", json=d)
assert r.status_code == 200, r.json
Expand Down