First commit of 1.8 upgrade

Author: agger-magenta

.gitignore

@@ -8,3 +8,5 @@ django-os2webscanner/django.os2webscanner.egg-info
 *.egg-info
 /webscanner_site/webscanner/local_settings.py
 /scrapy-webscanner/scanner/local_settings.py
+.ropeproject
+var/

django-os2webscanner/os2webscanner/forms.py

@@ -26,6 +26,7 @@ class ScannerForm(forms.ModelForm):
 
     class Meta:
         model = Scanner
+        fields = '__all__'
 
 
 class FileUploadForm(forms.Form):
@@ -34,13 +35,12 @@ class FileUploadForm(forms.Form):
 
     def validate_filetype(upload_file):
         extension = upload_file.name.split('.')[-1]
-        if not extension in ['odp', 'xls', 'xlsx', 'csv', 'pdf']:
+        if extension not in ['odp', 'xls', 'xlsx', 'csv', 'pdf']:
             raise ValidationError(
                 'Please upload a spreadsheet or a PDF file!'
             )
 
     scan_file = forms.FileField(label="Fil", validators=[validate_filetype])
-    #scan_file = forms.FileField(label="Fil")
     # CPR scan
     do_cpr_scan = forms.BooleanField(label="Scan CPR-numre", initial=True,
                                      required=False)
@@ -57,19 +57,19 @@ def validate_filetype(upload_file):
     name_replacement_text = forms.CharField(label="Erstat match med",
                                             required=False,
                                             initial="NAVN",
-                                           )
+                                            )
     # Address scan
     do_address_scan = forms.BooleanField(label="Scan adresser", initial=True,
-                                        required=False)
+                                         required=False)
     do_replace_address = forms.BooleanField(label="Erstat adresser",
                                             initial=False, required=False)
     address_replacement_text = forms.CharField(label="Erstat match med",
                                                required=False,
                                                initial="ADRESSE",
-                                              )
+                                               )
     # OCR scan
     do_ocr = forms.BooleanField(label="OCR scan", initial=False,
-                                     required=False)
+                                required=False)
     # Column list - ignored if left blank
     column_list = forms.RegexField(
         label="Scan kun kolonner",

django-os2webscanner/os2webscanner/migrations/0001_initial.py

@@ -68,17 +68,17 @@ class Organization(models.Model):
                                         editable=settings.DO_USE_GROUPS)
 
     name_whitelist = models.TextField(blank=True,
-                                       default="",
-                                       verbose_name='Godkendte navne')
+                                      default="",
+                                      verbose_name='Godkendte navne')
     name_blacklist = models.TextField(blank=True,
-                                       default="",
-                                       verbose_name='Sortlistede navne')
+                                      default="",
+                                      verbose_name='Sortlistede navne')
     address_whitelist = models.TextField(blank=True,
-                                       default="",
-                                       verbose_name='Godkendte adresser')
+                                         default="",
+                                         verbose_name='Godkendte adresser')
     address_blacklist = models.TextField(blank=True,
-                                       default="",
-                                       verbose_name='Sortlistede adresser')
+                                         default="",
+                                         verbose_name='Sortlistede adresser')
 
     def __unicode__(self):
         """Return the name of the organization."""
@@ -95,10 +95,9 @@ class UserProfile(models.Model):
     organization = models.ForeignKey(Organization,
                                      null=False,
                                      verbose_name='Organisation')
-    user = models.ForeignKey(User,
-                             unique=True,
-                             related_name='profile',
-                             verbose_name='Bruger')
+    user = models.OneToOneField(User,
+                                related_name='profile',
+                                verbose_name='Bruger')
     is_group_admin = models.BooleanField(default=False)
     is_upload_only = models.BooleanField(default=False)
 
@@ -117,7 +116,7 @@ class Group(models.Model):
     name = models.CharField(max_length=256, unique=True, verbose_name='Navn')
     contact_email = models.CharField(max_length=256, verbose_name='Email')
     contact_phone = models.CharField(max_length=256, verbose_name='Telefon')
-    user_profiles = models.ManyToManyField(UserProfile, null=True, blank=True,
+    user_profiles = models.ManyToManyField(UserProfile, blank=True,
                                            verbose_name='Brugere',
                                            related_name='groups')
     organization = models.ForeignKey(Organization,
@@ -219,8 +218,10 @@ def exclusion_rule_list(self):
     def root_url(self):
         """Return the root url of the domain."""
         url = self.url.replace('*.', '')
-        if (not self.url.startswith('http://') and not
-            self.url.startswith('https://')):
+        if (
+            not self.url.startswith('http://') and not
+            self.url.startswith('https://')
+        ):
             return 'http://%s/' % url
         else:
             return url
@@ -298,7 +299,7 @@ class Scanner(models.Model):
     organization = models.ForeignKey(Organization, null=False,
                                      verbose_name='Organisation')
     group = models.ForeignKey(Group, null=True, blank=True,
-                                     verbose_name='Gruppe')
+                              verbose_name='Gruppe')
     schedule = RecurrenceField(max_length=1024,
                                verbose_name='Planlagt afvikling')
     domains = models.ManyToManyField(Domain, related_name='scanners',
@@ -327,15 +328,14 @@ class Scanner(models.Model):
         verbose_name='Brug HEAD request'
     )
     do_collect_cookies = models.BooleanField(default=False,
-                                        verbose_name='Saml cookies')
+                                             verbose_name='Saml cookies')
     columns = models.CommaSeparatedIntegerField(max_length=128,
                                                 null=True,
                                                 blank=True)
     regex_rules = models.ManyToManyField(RegexRule,
                                          blank=True,
-                                         null=True,
                                          verbose_name='Regex regler')
-    recipients = models.ManyToManyField(UserProfile, null=True, blank=True,
+    recipients = models.ManyToManyField(UserProfile, blank=True,
                                         verbose_name='Modtagere')
 
     # Spreadsheet annotation and replacement parameters
@@ -434,7 +434,7 @@ def schedule_description(self):
     def has_active_scans(self):
         """Whether the scanner has active scans."""
         active_scanners = Scan.objects.filter(scanner=self, status__in=(
-                        Scan.NEW, Scan.STARTED)).count()
+            Scan.NEW, Scan.STARTED)).count()
         return active_scanners > 0
 
     @property
@@ -443,9 +443,9 @@ def has_valid_domains(self):
 
     # Run error messages
     ALREADY_RUNNING = ("Scanneren kunne ikke startes," +
-                        " fordi der allerede er en scanning i gang for den.")
+                       " fordi der allerede er en scanning i gang for den.")
     NO_VALID_DOMAINS = ("Scanneren kunne ikke startes," +
-                         " fordi den ikke har nogen gyldige domæner.")
+                        " fordi den ikke har nogen gyldige domæner.")
 
     def run(self, test_only=False, blocking=False, user=None):
         """Run a scan with the Scanner.
@@ -466,7 +466,7 @@ def run(self, test_only=False, blocking=False, user=None):
         scan = Scan.create(self)
         # Add user as recipient on scan
         if user:
-            scan.recipients.add(user.get_profile())
+            scan.recipients.add(user.profile)
         # Get path to run script
         SCRAPY_WEBSCANNER_DIR = os.path.join(base_dir, "scrapy-webscanner")
 
@@ -526,13 +526,14 @@ class Scan(models.Model):
                                          default="",
                                          verbose_name='Sortlistede navne')
     whitelisted_addresses = models.TextField(max_length=4096, blank=True,
-                                         default="",
-                                         verbose_name='Godkendte adresser')
-    blacklisted_addresses = models.TextField(max_length=4096, blank=True,
-                                         default="",
-                                         verbose_name='Sortlistede adresser')
+                                             default="",
+                                             verbose_name='Godkendte adresser')
+    blacklisted_addresses = models.TextField(
+        max_length=4096, blank=True,
+        default="",
+        verbose_name='Sortlistede adresser'
+    )
     domains = models.ManyToManyField(Domain,
-                                     null=True,
                                      verbose_name='Domæner')
     do_cpr_scan = models.BooleanField(default=True, verbose_name='CPR')
     do_name_scan = models.BooleanField(default=False, verbose_name='Navn')
@@ -558,16 +559,15 @@ class Scan(models.Model):
         verbose_name='Brug HEAD request'
     )
     do_collect_cookies = models.BooleanField(default=False,
-                                        verbose_name='Saml cookies')
+                                             verbose_name='Saml cookies')
 
     columns = models.CommaSeparatedIntegerField(max_length=128,
                                                 null=True,
                                                 blank=True)
     regex_rules = models.ManyToManyField(RegexRule,
                                          blank=True,
-                                         null=True,
                                          verbose_name='Regex regler')
-    recipients = models.ManyToManyField(UserProfile, null=True, blank=True)
+    recipients = models.ManyToManyField(UserProfile, blank=True)
 
     # Spreadsheet annotation and replacement parameters
 
@@ -810,15 +810,19 @@ def save(self, *args, **kwargs):
         used by the scan.
         """
         # Pre-save stuff
-        if (self.status in [Scan.DONE, Scan.FAILED] and
-            (self._old_status != self.status)):
+        if (
+            self.status in [Scan.DONE, Scan.FAILED] and
+            (self._old_status != self.status)
+        ):
             self.end_time = datetime.datetime.now()
         # Actual save
         super(Scan, self).save(*args, **kwargs)
         # Post-save stuff
 
-        if (self.status in [Scan.DONE, Scan.FAILED] and
-            (self._old_status != self.status)):
+        if (
+            self.status in [Scan.DONE, Scan.FAILED] and
+            (self._old_status != self.status)
+        ):
             # Send email
             from os2webscanner.utils import notify_user
             try:
@@ -840,7 +844,7 @@ def cleanup_finished_scan(self, log=False):
             if pending_items.exists():
                 print "Deleting %d remaining conversion queue items from " \
                       "finished scan %s" % (
-                    pending_items.count(), self)
+                          pending_items.count(), self)
 
         pending_items.delete()
 
@@ -940,7 +944,7 @@ class Url(models.Model):
                                       verbose_name='Status ' + 'Message')
     referrers = models.ManyToManyField("ReferrerUrl",
                                        related_name='linked_urls',
-                                       null=True, verbose_name='Referrers')
+                                       verbose_name='Referrers')
 
     def __unicode__(self):
         """Return the URL."""
@@ -1064,9 +1068,9 @@ class UrlLastModified(models.Model):
 
     url = models.CharField(max_length=2048, verbose_name='Url')
     last_modified = models.DateTimeField(blank=True, null=True,
-                                    verbose_name='Last-modified')
+                                         verbose_name='Last-modified')
     links = models.ManyToManyField("self", symmetrical=False,
-                                    null=True, verbose_name='Links')
+                                   verbose_name='Links')
     scanner = models.ForeignKey(Scanner, null=False, verbose_name='Scanner')
 
     def __unicode__(self):
@@ -1085,15 +1089,15 @@ class Summary(models.Model):
     schedule = RecurrenceField(max_length=1024,
                                verbose_name='Planlagt afvikling')
     last_run = models.DateTimeField(blank=True, null=True,
-                                      verbose_name='Sidste kørsel')
-    recipients = models.ManyToManyField(UserProfile, null=True, blank=True,
+                                    verbose_name='Sidste kørsel')
+    recipients = models.ManyToManyField(UserProfile, blank=True,
                                         verbose_name="Modtagere")
-    scanners = models.ManyToManyField(Scanner, null=True, blank=True,
+    scanners = models.ManyToManyField(Scanner, blank=True,
                                       verbose_name="Scannere")
     organization = models.ForeignKey(Organization, null=False,
                                      verbose_name='Organisation')
     group = models.ForeignKey(Group, null=True, blank=True,
-                                     verbose_name='Gruppe')
+                              verbose_name='Gruppe')
     do_email_recipients = models.BooleanField(default=False,
                                               verbose_name="Udsend mails")
 
@@ -1124,7 +1128,7 @@ class Md5Sum(models.Model):
 
     class Meta:
         unique_together = ('md5', 'is_cpr_scan', 'is_check_mod11',
-                        'is_ignore_irrelevant', 'organization')
+                           'is_ignore_irrelevant', 'organization')
 
     def __unicode__(self):
         return u"{0}: {1}".format(self.organization.name, self.md5)

django-os2webscanner/os2webscanner/migrations/__init__.py

@@ -45,7 +45,7 @@ def scan_urls(username, password, urls, params={}):
     user = authenticate(username=username, password=password)
     if not user:
         raise RuntimeError("Wrong username or password!")
-    scan = do_scan(user, urls, params)
+    do_scan(user, urls, params)
 
     return do_scan_urls(user, urls, params)
 
@@ -142,9 +142,11 @@ def get_report(username, password, report_url):
     writer.writerow(e([u'Starttidspunkt', u'Sluttidspunkt', u'Status',
                     u'Totalt antal matches']))
     # Print summary
-    writer.writerow(e([str(scan.start_time),
-        str(scan.end_time), scan.get_status_display(),
-        str(len(all_matches))]))
+    writer.writerow(
+        e([str(scan.start_time),
+           str(scan.end_time), scan.get_status_display(),
+           str(len(all_matches))])
+    )
     # Print match header
     writer.writerow(e([u'URL', u'Regel', u'Match', u'Følsomhed']))
     for match in all_matches:

django-os2webscanner/os2webscanner/models.py

@@ -41,12 +41,12 @@
       <div class="collapse navbar-collapse">
         <ul class="nav navbar-nav">
             {% if user.is_authenticated %}
-            {% if not user.get_profile.is_upload_only %}
+            {% if not user.profile.is_upload_only %}
           <li {% block scanners_active %}{% endblock %}>
             <a href="/scanners/">Scannere</a>
           </li>
-          {% if user.get_profile.is_groups_enabled %}
-          {% if user.is_superuser or user.get_profile.is_group_admin %}
+          {% if user.profile.is_groups_enabled %}
+          {% if user.is_superuser or user.profile.is_group_admin %}
           <li {% block groups_active %}{% endblock %}>
             <a href="/groups/">Grupper</a>
           </li>
@@ -81,7 +81,7 @@
                 <li>
                 <a href="/system/upload_file/">Scan regneark</a>
                 </li>
-                {% if not user.get_profile.is_upload_only %}
+                {% if not user.profile.is_upload_only %}
                 <li><a href="/organization/">Oplæring</a></li>
                 {% endif %}
                 <li>

django-os2webscanner/os2webscanner/rpc.py

@@ -37,7 +37,7 @@ class ScannerTest(TestCase):
     # Internet connection! !!!
 
     @classmethod
-    def setUpClass(cls):
+    def setUpTestData(cls):
         """Setup some data to test with."""
         # Don't change the order of these, because Magenta needs
         # pk = 2 to pass the validation test
@@ -86,7 +86,7 @@ def pep8_test(filepath):
     """Run a pep8 test on the filepath."""
     def do_test(self):
         # print "PATH:", filepath
-        arglist = ['--exclude=lib', filepath]
+        arglist = ['--exclude=lib,migrations', filepath]
         pep8.process_options(arglist)
         pep8.input_dir(filepath)
         output = pep8.get_statistics()

django-os2webscanner/os2webscanner/templates/os2webscanner/base.html

@@ -88,7 +88,7 @@ def do_scan(user, urls, params={}, blocking=False, visible=False):
     parameters and values. Defaults are used for unspecified parameters.
     """
     scanner = models.Scanner()
-    scanner.organization = user.get_profile().organization
+    scanner.organization = user.profile.organization
     scanner.name = user.username + '-' + str(time.time())
     scanner.do_run_synchronously = True
     scanner.do_last_modified_check = False