Skip to content

Commit c127a11

Browse files
dimitri-yatsenkodimitri-yatsenko
authored
Merge pull request #713 from guzman-raphael/dev-details
Update Log table
2 parents 94c2eb8 + 4407ba1 commit c127a11

File tree

7 files changed

+159
-11
lines changed

7 files changed

+159
-11
lines changed

‎.gitignore

+3-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ build/
1919
./tests/dj-store/*
2020
*.log
2121
*.env
22-
local-docker-compose.yml
23-
notebooks/*
22+
docker-compose.yml
23+
notebook
24+
.vscode
2425
__main__.py
2526
jupyter_custom.js

‎.travis.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ main: &main
1616
dist: xenial # precise, trusty, xenial, bionic
1717
language: shell
1818
script:
19-
- docker-compose -f LNX-docker-compose.yml up --build --exit-code-from dj
19+
- docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
2020
jobs:
2121
include:
2222
- <<: *main

‎LNX-docker-compose.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ x-net: &net
33
networks:
44
- main
55
services:
6-
dj:
6+
app:
77
<<: *net
88
image: datajoint/pydev:${PY_VER}-alpine${ALPINE_VER}
99
depends_on:

‎README.md

+40
Original file line numberDiff line numberDiff line change
@@ -98,3 +98,43 @@ A number of labs are currently adopting DataJoint and we are quickly getting the
9898
* https://docs.datajoint.io -- up-to-date documentation
9999
* https://tutorials.datajoint.io -- step-by-step tutorials
100100
* https://catalog.datajoint.io -- catalog of example pipelines
101+
102+
## Running Tests Locally
103+
104+
105+
* Create an `.env` with desired development environment values e.g.
106+
``` sh
107+
PY_VER=3.7
108+
ALPINE_VER=3.10
109+
MYSQL_VER=5.7
110+
MINIO_VER=RELEASE.2019-09-26T19-42-35Z
111+
UID=1000
112+
GID=1000
113+
```
114+
* `cp local-docker-compose.yml docker-compose.yml`
115+
* `docker-compose up -d` (Note configured `JUPYTER_PASSWORD`)
116+
* Select a means of running Tests e.g. Docker Terminal, or Local Terminal (see bottom)
117+
* Run desired tests. Some examples are as follows:
118+
119+
| Use Case | Shell Code |
120+
| ---------------------------- | ------------------------------------------------------------------------------ |
121+
| Run all tests | `nosetests -vsw tests --with-coverage --cover-package=datajoint` |
122+
| Run one specific class test | `nosetests -vs --tests=tests.test_fetch:TestFetch.test_getattribute_for_fetch1` |
123+
| Run one specific basic test | `nosetests -vs --tests=tests.test_external_class:test_insert_and_fetch` |
124+
125+
126+
### Launch Docker Terminal
127+
* Shell into `datajoint-python_app_1` i.e. `docker exec -it datajoint-python_app_1 sh`
128+
129+
130+
### Launch Local Terminal
131+
* See `datajoint-python_app` environment variables in `local-docker-compose.yml`
132+
* Launch local terminal
133+
* `export` environment variables in shell
134+
* Add entry in `/etc/hosts` for `127.0.0.1 fakeminio.datajoint.io`
135+
136+
137+
### Launch Jupyter Notebook for Interactive Use
138+
* Navigate to `localhost:8888`
139+
* Input Jupyter password
140+
* Launch a notebook i.e. `New > Python 3`

‎datajoint/declare.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
EXTERNAL_TABLE_ROOT = '~external'
1717

1818
TYPE_PATTERN = {k: re.compile(v, re.I) for k, v in dict(
19-
INTEGER=r'((tiny|small|medium|big|)int|integer)(\s*\(.+\))?(\s+unsigned)?(\s+auto_increment)?$',
19+
INTEGER=r'((tiny|small|medium|big|)int|integer)(\s*\(.+\))?(\s+unsigned)?(\s+auto_increment)?|serial$',
2020
DECIMAL=r'(decimal|numeric)(\s*\(.+\))?(\s+unsigned)?$',
2121
FLOAT=r'(double|float|real)(\s*\(.+\))?(\s+unsigned)?$',
2222
STRING=r'(var)?char\s*\(.+\)$',
@@ -443,7 +443,8 @@ def compile_attribute(line, in_key, foreign_key_sql, context):
443443
match['default'] = 'DEFAULT NULL' # nullable attributes default to null
444444
else:
445445
if match['default']:
446-
quote = match['default'].upper() not in CONSTANT_LITERALS and match['default'][0] not in '"\''
446+
quote = (match['default'].split('(')[0].upper() not in CONSTANT_LITERALS
447+
and match['default'][0] not in '"\'')
447448
match['default'] = 'NOT NULL DEFAULT ' + ('"%s"' if quote else "%s") % match['default']
448449
else:
449450
match['default'] = 'NOT NULL'

‎datajoint/table.py

+6-5
Original file line numberDiff line numberDiff line change
@@ -683,12 +683,13 @@ def __init__(self, arg, database=None):
683683
self.database = database
684684
self._connection = arg
685685
self._definition = """ # event logging table for `{database}`
686-
timestamp = CURRENT_TIMESTAMP : timestamp
686+
id :int unsigned auto_increment # event order id
687687
---
688-
version :varchar(12) # datajoint version
689-
user :varchar(255) # user@host
690-
host="" :varchar(255) # system hostname
691-
event="" :varchar(255) # custom message
688+
timestamp = CURRENT_TIMESTAMP : timestamp # event timestamp
689+
version :varchar(12) # datajoint version
690+
user :varchar(255) # user@host
691+
host="" :varchar(255) # system hostname
692+
event="" :varchar(255) # event message
692693
""".format(database=database)
693694

694695
if not self.is_declared:

‎local-docker-compose.yml

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
version: '2.2'
2+
x-net: &net
3+
networks:
4+
- main
5+
services:
6+
app:
7+
<<: *net
8+
image: datajoint/pydev:${PY_VER}-alpine${ALPINE_VER}
9+
depends_on:
10+
db:
11+
condition: service_healthy
12+
minio:
13+
condition: service_healthy
14+
environment:
15+
- DJ_HOST=db
16+
- DJ_USER=root
17+
- DJ_PASS=simple
18+
- DJ_TEST_HOST=db
19+
- DJ_TEST_USER=datajoint
20+
- DJ_TEST_PASSWORD=datajoint
21+
# If running tests locally, make sure to add entry in /etc/hosts for 127.0.0.1 fakeminio.datajoint.io
22+
- S3_ENDPOINT=fakeminio.datajoint.io:9000
23+
- S3_ACCESS_KEY=datajoint
24+
- S3_SECRET_KEY=datajoint
25+
- S3_BUCKET=datajoint-test
26+
- PYTHON_USER=dja
27+
- JUPYTER_PASSWORD=datajoint
28+
- DISPLAY
29+
working_dir: /src
30+
command: >
31+
/bin/sh -c
32+
"
33+
pip install --user nose nose-cov coveralls ptvsd .;
34+
pip freeze | grep datajoint;
35+
## You may run the below tests once sh'ed into container i.e. docker exec -it datajoint-python_app_1 sh
36+
# nosetests -vsw tests --with-coverage --cover-package=datajoint; #run all tests
37+
# nosetests -vs --tests=tests.test_external_class:test_insert_and_fetch; #run specific basic test
38+
# nosetests -vs --tests=tests.test_fetch:TestFetch.test_getattribute_for_fetch1; #run specific Class test
39+
## Interactive Jupyter Notebook environment
40+
jupyter notebook &
41+
## Remote debugger
42+
while true;
43+
do python -m ptvsd --host 0.0.0.0 --port 5678 --wait .;
44+
sleep 2;
45+
done;
46+
"
47+
ports:
48+
- "8888:8888"
49+
- "5678:5678"
50+
user: ${UID}:${GID}
51+
volumes:
52+
- .:/src
53+
- /tmp/.X11-unix:/tmp/.X11-unix:rw
54+
# Additional mounted notebooks may go here
55+
# - ./notebook:/home/dja/notebooks
56+
# - ../dj-python-101/ch1:/home/dja/tutorials
57+
db:
58+
<<: *net
59+
image: datajoint/mysql:$MYSQL_VER
60+
environment:
61+
- MYSQL_ROOT_PASSWORD=simple
62+
ports:
63+
- "3306:3306"
64+
# To persist MySQL data
65+
# volumes:
66+
# - ./mysql/data:/var/lib/mysql
67+
minio:
68+
<<: *net
69+
environment:
70+
- MINIO_ACCESS_KEY=datajoint
71+
- MINIO_SECRET_KEY=datajoint
72+
image: minio/minio:$MINIO_VER
73+
# To persist MinIO data and config
74+
# volumes:
75+
# - ./minio/data:/data
76+
# - ./minio/config:/root/.minio
77+
command: server /data
78+
healthcheck:
79+
test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"]
80+
timeout: 5s
81+
retries: 60
82+
interval: 1s
83+
fakeminio.datajoint.io:
84+
<<: *net
85+
image: nginx:alpine
86+
environment:
87+
- URL=datajoint.io
88+
- SUBDOMAINS=fakeminio
89+
- MINIO_SERVER=http://minio:9000
90+
entrypoint: /entrypoint.sh
91+
healthcheck:
92+
test: wget --quiet --tries=1 --spider https://fakeminio.datajoint.io:443/minio/health/live || exit 1
93+
timeout: 5s
94+
retries: 300
95+
interval: 1s
96+
ports:
97+
- "9000:9000"
98+
- "443:443"
99+
volumes:
100+
- ./tests/nginx/base.conf:/base.conf
101+
- ./tests/nginx/entrypoint.sh:/entrypoint.sh
102+
- ./tests/nginx/fullchain.pem:/certs/fullchain.pem
103+
- ./tests/nginx/privkey.pem:/certs/privkey.pem
104+
networks:
105+
main:

0 commit comments

Comments
 (0)