From e1b26c90e0dbe2421280d5ea738052e50c44d399 Mon Sep 17 00:00:00 2001
From: Ismail Muhammed Ismail <ismailmd.dev@gmail.com>
Date: Tue, 24 Feb 2026 14:51:37 +0200
Subject: [PATCH] feat: Add GitHub API rate limit handling to temporarily pause
 processing of GitHub URIs.

Signed-off-by: Ismail Muhammed Ismail <ismailmd.dev@gmail.com>
---
 minecode/miners/github.py | 11 +++++++++++
 minecode/models.py        | 11 ++++++++++-
 minecode/utils.py         | 28 ++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/minecode/miners/github.py b/minecode/miners/github.py
index bde7e93f..88f6b139 100644
--- a/minecode/miners/github.py
+++ b/minecode/miners/github.py
@@ -94,7 +94,18 @@ def fetch(self, uri, timeout=None):
         The main idea is to fetch download_url...
         """
         full_name = uri.replace("https://api.github.com/repos/", "")
+
         g = Github()
+
+        rate_limit = g.get_rate_limit().core
+        if rate_limit.remaining == 0:
+            import calendar
+            from minecode.utils import set_github_rate_limit
+
+            reset_epoch = calendar.timegm(rate_limit.reset.timetuple())
+            set_github_rate_limit(reset_epoch)
+            raise Exception("GitHub API Rate Limit Exceeded: Bailing to allow queue to continue")
+
         repo = g.get_repo(full_name)
 
         common_data = dict(
diff --git a/minecode/models.py b/minecode/models.py
index a243b21b..7e40ea9d 100644
--- a/minecode/models.py
+++ b/minecode/models.py
@@ -300,7 +300,16 @@ def get_next_visitable(self):
         # inconsistent view of the data, so this is not suitable for
         # general purpose work, but can be used to avoid lock contention
         # with multiple consumers accessing a queue-like table.
-        resource_uri = self.get_visitables().select_for_update(skip_locked=True).first()
+
+        visitables = self.get_visitables()
+
+        from minecode.utils import is_github_rate_limit_active
+
+        if is_github_rate_limit_active():
+            # Exclude all URIs that start with the GitHub API
+            visitables = visitables.exclude(uri__startswith="https://api.github.com")
+
+        resource_uri = visitables.select_for_update(skip_locked=True).first()
         if not resource_uri:
             return
 
diff --git a/minecode/utils.py b/minecode/utils.py
index 16ac1513..5ee0c929 100644
--- a/minecode/utils.py
+++ b/minecode/utils.py
@@ -404,3 +404,31 @@ def get_webhook_url(view_name, user_uuid):
     site_url = settings.SITE_URL.rstrip("/")
     webhook_url = site_url + target_url
     return webhook_url
+
+
+def is_github_rate_limit_active():
+    """
+    Check if the GitHub API rate limit is currently active.
+    Returns True if the limit is active, False otherwise.
+    """
+    from django.core.cache import cache
+    import time
+
+    gh_reset = cache.get("github_limit_reset")
+    if gh_reset and time.time() < gh_reset:
+        return True
+    return False
+
+
+def set_github_rate_limit(reset_epoch=None):
+    """
+    Set the GitHub API rate limit in the cache using the provided reset epoch timestamp.
+    If no reset timestamp is provided, defaults to 1 hour from now.
+    """
+    from django.core.cache import cache
+    import time
+
+    if reset_epoch:
+        cache.set("github_limit_reset", int(reset_epoch), timeout=3600 * 24)
+    else:
+        cache.set("github_limit_reset", int(time.time()) + 3600, timeout=3600)