Task/GitHub codebases

src/gurubase-backend/backend/core/admin.py

@@ -165,7 +165,7 @@ def guru_type(self, obj):
 
 @admin.register(GuruType)
 class GuruTypeAdmin(admin.ModelAdmin):
-    list_display = ['id', 'slug', 'active', 'has_sitemap_added_questions', 'icon_url', 'stackoverflow_tag', 'domain_knowledge', 'colors', 'custom', 'maintainers_list', 'github_repos', 'text_embedding_model', 'code_embedding_model', 'date_created', 'date_updated', 'github_details_updated_date']
+    list_display = ['id', 'slug', 'active', 'has_sitemap_added_questions', 'icon_url', 'stackoverflow_tag', 'domain_knowledge', 'colors', 'custom', 'maintainers_list', 'text_embedding_model', 'code_embedding_model', 'date_created', 'date_updated']
     search_fields = ['id', 'slug', 'icon_url', 'stackoverflow_tag', 'domain_knowledge', 'date_created', 'date_updated', 'maintainers__email']
     list_filter = ('active', 'custom', 'has_sitemap_added_questions', 'text_embedding_model', 'code_embedding_model')
     ordering = ('-id',)

src/gurubase-backend/backend/core/data_sources.py

@@ -521,38 +521,43 @@ def create(self, guru_type_object, ticket_url):
                 'message': str(e)
             }
 
-class GitHubRepoStrategy(DataSourceStrategy):
-    def create(self, guru_type_object, repo_url):
+class GitHubStrategy(DataSourceStrategy):
+    def create(self, guru_type_object, repo):
+        url = repo['url']
+        glob_pattern = repo['glob_pattern']
+        glob_include = repo['include_glob']
         try:
-            # Create the data source
             data_source = DataSource.objects.create(
                 type=DataSource.Type.GITHUB_REPO,
                 guru_type=guru_type_object,
-                url=repo_url
+                url=url,
+                github_glob_pattern=glob_pattern,
+                github_glob_include=glob_include
             )
-
-            data_source.save()
-
             return {
-                'type': 'GITHUB_REPO',
-                'url': repo_url,
+                'type': 'GitHub',
+                'url': url,
                 'status': 'success',
                 'id': data_source.id,
-                'title': data_source.title
+                'title': data_source.title,
+                'github_glob_pattern': glob_pattern,
+                'github_glob_include': glob_include
             }
         except DataSourceExists as e:
             return {
-                'type': 'GITHUB_REPO',
-                'url': repo_url,
+                'type': 'GitHub',
+                'url': url,
                 'status': 'exists',
                 'id': e.args[0]['id'],
-                'title': e.args[0]['title']
+                'title': e.args[0]['title'],
+                'github_glob_pattern': e.args[0]['github_glob_pattern'],
+                'github_glob_include': e.args[0]['github_glob_include']
             }
         except Exception as e:
-            logger.error(f'Error processing GitHub repository {repo_url}: {traceback.format_exc()}')
+            logger.error(f'Error processing GitHub repository {url}: {traceback.format_exc()}')
             return {
-                'type': 'GITHUB_REPO',
-                'url': repo_url,
+                'type': 'GitHub',
+                'url': url,
                 'status': 'error',
                 'message': str(e)
             }

src/gurubase-backend/backend/core/migrations/0081_remove_gurutype_github_repos_and_more.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.18 on 2025-04-21 14:44
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0080_settings_gurubase_url'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='gurutype',
+            name='github_repos',
+        ),
+        migrations.RemoveField(
+            model_name='gurutype',
+            name='index_repo',
+        ),
+    ]

src/gurubase-backend/backend/core/migrations/0082_datasource_github_details_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.18 on 2025-04-24 08:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0081_remove_gurutype_github_repos_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='datasource',
+            name='github_details',
+            field=models.JSONField(blank=True, default=dict, null=True),
+        ),
+        migrations.AddField(
+            model_name='datasource',
+            name='github_details_updated_date',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

src/gurubase-backend/backend/core/migrations/0083_remove_gurutype_github_details_and_more.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.18 on 2025-04-24 09:34
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0082_datasource_github_details_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='gurutype',
+            name='github_details',
+        ),
+        migrations.RemoveField(
+            model_name='gurutype',
+            name='github_details_updated_date',
+        ),
+    ]

src/gurubase-backend/backend/core/models.py

@@ -304,9 +304,6 @@ class EmbeddingModel(models.TextChoices):
     name = models.CharField(max_length=50, blank=True, null=True)
     maintainers = models.ManyToManyField(User, blank=True, related_name='maintained_guru_types')
     stackoverflow_tag = models.CharField(max_length=100, blank=True, null=True)
-    github_repos = models.JSONField(default=list, blank=True)
-    github_details = models.JSONField(default=dict, blank=True, null=False)
-    github_details_updated_date = models.DateTimeField(null=True, blank=True)
     colors = models.JSONField(default=dict, blank=True, null=False)
     icon_url = models.CharField(max_length=2000, default="", blank=True, null=True)
     ogimage_url = models.URLField(max_length=2000, default="", blank=True, null=True)  # question
@@ -320,7 +317,6 @@ class EmbeddingModel(models.TextChoices):
     domain_knowledge = models.TextField(default='', blank=True, null=True)
     custom_instruction_prompt = models.TextField(default='', blank=True, null=True)
     has_sitemap_added_questions = models.BooleanField(default=False)
-    index_repo = models.BooleanField(default=True)
     # GitHub repository limits
     github_repo_count_limit = models.IntegerField(default=1)
     github_file_count_limit_per_repo_soft = models.IntegerField(default=1000)  # Warning threshold
@@ -386,19 +382,12 @@ def save(self, *args, **kwargs):
         if self.slug == '':
             raise ValidationError({'msg': 'Guru type name cannot be empty'})
 
-        unique_github_repos = set(self.github_repos)
-
-        if settings.ENV != 'selfhosted' and len(unique_github_repos) > self.github_repo_count_limit:
-            raise ValidationError({'msg': f'You have reached the maximum number ({self.github_repo_count_limit}) of GitHub repositories for this guru type.'})
-
         if settings.ENV == 'selfhosted':
             if self.text_embedding_model == GuruType.EmbeddingModel.IN_HOUSE:
                 raise ValidationError({'msg': 'In-house embedding model is not allowed in selfhosted environment.'})
             if self.code_embedding_model == GuruType.EmbeddingModel.IN_HOUSE:
                 raise ValidationError({'msg': 'In-house embedding model is not allowed in selfhosted environment.'})
 
-        self.github_repos = list(unique_github_repos)
-
         super().save(*args, **kwargs)
 
     def generate_widget_id(self, domain_url):
@@ -481,7 +470,7 @@ def ready(self):
 
         return non_processed_count == 0 and non_written_count == 0
 
-    def check_datasource_limits(self, user, file=None, website_urls_count=0, youtube_urls_count=0, github_urls_count=0, jira_urls_count=0, zendesk_urls_count=0, confluence_urls_count=0):
+    def check_datasource_limits(self, user, file=None, website_urls_count=0, youtube_urls_count=0, jira_urls_count=0, zendesk_urls_count=0, confluence_urls_count=0, github_repos_count=0):
         """
         Checks if adding a new datasource would exceed the limits for this guru type.
         Returns (bool, str) tuple - (is_allowed, error_message)
@@ -510,11 +499,6 @@ def check_datasource_limits(self, user, file=None, website_urls_count=0, youtube
             type=DataSource.Type.YOUTUBE
         ).count()
 
-        github_count = DataSource.objects.filter(
-            guru_type=self,
-            type=DataSource.Type.GITHUB_REPO
-        ).count()
-
         jira_count = DataSource.objects.filter(
             guru_type=self,
             type=DataSource.Type.JIRA
@@ -535,6 +519,12 @@ def check_datasource_limits(self, user, file=None, website_urls_count=0, youtube
             guru_type=self,
             type=DataSource.Type.PDF
         )
+
+        github_repo_count = DataSource.objects.filter(
+            guru_type=self,
+            type=DataSource.Type.GITHUB_REPO
+        ).count()
+
         total_pdf_mb = 0
         for source in pdf_sources:
             if source.file:
@@ -549,7 +539,7 @@ def check_datasource_limits(self, user, file=None, website_urls_count=0, youtube
             return False, f"YouTube video limit ({self.youtube_count_limit}) reached"
 
         # Check GitHub repo limit
-        if (github_count + github_urls_count) > self.github_repo_count_limit:
+        if (github_repo_count + github_repos_count) > self.github_repo_count_limit:
             return False, f"GitHub repository limit ({self.github_repo_count_limit}) reached"
 
         # Check Jira issue limit
@@ -558,7 +548,7 @@ def check_datasource_limits(self, user, file=None, website_urls_count=0, youtube
 
         # Check Zendesk ticket limit
         if (zendesk_count + zendesk_urls_count) > self.zendesk_count_limit:
-            return False, f"Zendesk ticket limit ({self.zendesk_count_limit}) reached"
+            return False, f"Zendesk ticket/article limit ({self.zendesk_count_limit}) reached"
 
         # Check Confluence page limit
         if (confluence_count + confluence_urls_count) > self.confluence_count_limit:
@@ -639,7 +629,7 @@ class Type(models.TextChoices):
         ZENDESK = "ZENDESK"
         CONFLUENCE = "CONFLUENCE"
 
-    class Status(models.TextChoices):
+    class Status(models.TextChoices): # This is the data retrieval status. It is not the readiness of the data source as it may not be written to milvus yet.
         NOT_PROCESSED = "NOT_PROCESSED"
         SUCCESS = "SUCCESS"
         FAIL = "FAIL"
@@ -680,14 +670,16 @@ class Status(models.TextChoices):
     final_summarization_created = models.BooleanField(default=False)
 
     default_branch = models.CharField(max_length=100, null=True, blank=True)  # Only used for Github Repos
+    github_details = models.JSONField(default=dict, blank=True, null=True)  # For storing GitHub repository details
+    github_details_updated_date = models.DateTimeField(null=True, blank=True)  # When GitHub details were last updated
 
     private = models.BooleanField(default=False)
 
-    last_reindex_date = models.DateTimeField(auto_now_add=True, null=True, blank=True)
+    last_reindex_date = models.DateTimeField(auto_now_add=True, null=True, blank=True) # Set when reindex is manually done
     reindex_count = models.IntegerField(default=0)
 
     scrape_tool = models.CharField(max_length=100, null=True, blank=True)
-    last_successful_index_date = models.DateTimeField(null=True, blank=True)
+    last_successful_index_date = models.DateTimeField(null=True, blank=True) # Set when github repo is indexed/reindexed successfully
     github_glob_include = models.BooleanField(default=True)
     github_glob_pattern = models.CharField(max_length=100, null=True, blank=True)
 
@@ -1109,9 +1101,18 @@ def reindex(self):
 
         if self.type == DataSource.Type.GITHUB_REPO:
             self.content = ''
-
-        self.save()
-        self.delete_from_milvus()
+            self.save()
+
+            # Import here to avoid circular imports
+            from core.tasks import update_github_repositories
+            # Delay the task to process this specific GitHub repository
+            update_github_repositories.delay(
+                guru_type_slug=self.guru_type.slug,
+                repo_url=self.url
+            )
+        else:
+            self.save()
+            self.delete_from_milvus()
 
 
 class FeaturedDataSource(models.Model):

src/gurubase-backend/backend/core/serializers.py

@@ -1,5 +1,5 @@
 from rest_framework import serializers
-from core.models import WidgetId, Binge, DataSource, GuruType, Question, FeaturedDataSource, APIKey, Settings, CrawlState
+from core.models import GithubFile, WidgetId, Binge, DataSource, GuruType, Question, FeaturedDataSource, APIKey, Settings, CrawlState
 from core.gcp import replace_media_root_with_nginx_base_url
 from django.conf import settings
 
@@ -73,7 +73,7 @@ class Meta:
 class DataSourceSerializer(serializers.ModelSerializer):
     class Meta:
         model = DataSource
-        exclude = ['file', 'doc_ids', 'content', 'guru_type']
+        exclude = ['file', 'doc_ids', 'content', 'guru_type', 'last_reindex_date']
 
     def to_representation(self, instance):
         from core.utils import format_github_repo_error
@@ -88,6 +88,9 @@ def to_representation(self, instance):
         if instance.type == DataSource.Type.GITHUB_REPO:
             if instance.error:
                 repr['error'] = format_github_repo_error(instance.error, instance.user_error)
+
+            repr['file_count'] = GithubFile.objects.filter(data_source=instance, in_milvus=True).count()
+            repr['last_reindex_date'] = instance.last_successful_index_date
 
         return repr