Add Github API rate limit handling

minecode/miners/github.py

@@ -94,7 +94,18 @@ def fetch(self, uri, timeout=None):
         The main idea is to fetch download_url...
         """
         full_name = uri.replace("https://api.github.com/repos/", "")
+
         g = Github()
+
+        rate_limit = g.get_rate_limit().core
+        if rate_limit.remaining == 0:
+            import calendar
+            from minecode.utils import set_github_rate_limit
+
+            reset_epoch = calendar.timegm(rate_limit.reset.timetuple())
+            set_github_rate_limit(reset_epoch)
+            raise Exception("GitHub API Rate Limit Exceeded: Bailing to allow queue to continue")
+
         repo = g.get_repo(full_name)
 
         common_data = dict(

minecode/models.py

@@ -300,7 +300,16 @@ def get_next_visitable(self):
         # inconsistent view of the data, so this is not suitable for
         # general purpose work, but can be used to avoid lock contention
         # with multiple consumers accessing a queue-like table.
-        resource_uri = self.get_visitables().select_for_update(skip_locked=True).first()
+
+        visitables = self.get_visitables()
+
+        from minecode.utils import is_github_rate_limit_active
+
+        if is_github_rate_limit_active():
+            # Exclude all URIs that start with the GitHub API
+            visitables = visitables.exclude(uri__startswith="https://api.github.com")
+
+        resource_uri = visitables.select_for_update(skip_locked=True).first()
         if not resource_uri:
             return
 

minecode/utils.py

@@ -404,3 +404,31 @@ def get_webhook_url(view_name, user_uuid):
     site_url = settings.SITE_URL.rstrip("/")
     webhook_url = site_url + target_url
     return webhook_url
+
+
+def is_github_rate_limit_active():
+    """
+    Check if the GitHub API rate limit is currently active.
+    Returns True if the limit is active, False otherwise.
+    """
+    from django.core.cache import cache
+    import time
+
+    gh_reset = cache.get("github_limit_reset")
+    if gh_reset and time.time() < gh_reset:
+        return True
+    return False
+
+
+def set_github_rate_limit(reset_epoch=None):
+    """
+    Set the GitHub API rate limit in the cache using the provided reset epoch timestamp.
+    If no reset timestamp is provided, defaults to 1 hour from now.
+    """
+    from django.core.cache import cache
+    import time
+
+    if reset_epoch:
+        cache.set("github_limit_reset", int(reset_epoch), timeout=3600 * 24)
+    else:
+        cache.set("github_limit_reset", int(time.time()) + 3600, timeout=3600)