From c79eb8f3cb2302482c79a293e1d0854ce0e24c8c Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 30 Apr 2019 15:43:54 +0100 Subject: [PATCH 001/183] Add "grid" db details and rename other to 'cloud' --- monitoring/publishing/views.py | 2 +- monitoring/settings.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 81d1656..7945f95 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -22,7 +22,7 @@ def list(self, request): print last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20) if last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): print 'Out of date' - fetchset = VAnonCloudRecord.objects.using('repository').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID), CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") + fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID), CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") for f in fetchset: CloudSite.objects.update_or_create(defaults={'script': f.CloudType, 'updated': f.UpdateTime}, name=f.SiteName) else: diff --git a/monitoring/settings.py b/monitoring/settings.py index d76e5d1..649ae2d 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -89,7 +89,14 @@ 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), }, - 'repository': { + 'grid': { + 'ENGINE': 'django.db.backends.mysql', + 'HOST': 'localhost', + 'PORT': '3306', + 'NAME': 'django_test', + 'USER': 'root', + }, + 'cloud': { 'ENGINE': 'django.db.backends.mysql', 'HOST': 'localhost', 'PORT': '3306', From f798f30915e1ae40123b4d9af0b18693c5fba718 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 30 Apr 2019 16:51:04 +0100 Subject: [PATCH 002/183] Change ModelViewSet to ReadOnly version --- monitoring/publishing/views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 7945f95..4558524 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -11,8 +11,7 @@ from models import CloudSite, VAnonCloudRecord from serializers import CloudSiteSerializer - -class CloudSiteViewSet(viewsets.ModelViewSet): +class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet): queryset = CloudSite.objects.all() serializer_class = CloudSiteSerializer template_name = 'cloudsites.html' From 58c79ddb783a84c973fb4033e2235a28e9d1e18a Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 30 Apr 2019 16:54:14 +0100 Subject: [PATCH 003/183] Add basic single site retrieve view method --- monitoring/publishing/views.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 4558524..5aaaf89 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -32,3 +32,21 @@ def list(self, request): if type(request.accepted_renderer) is TemplateHTMLRenderer: response.data = {'sites': response.data, 'last_fetched': last_fetched} return response + + def retrieve(self, request, pk=None): + last_fetched = CloudSite.objects.aggregate(Max('fetched'))['fetched__max'] + print last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20) + if last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): + print 'Out of date' + fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID), CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") + for f in fetchset: + CloudSite.objects.update_or_create(defaults={'script': f.CloudType, 'updated': f.UpdateTime}, name=f.SiteName) + else: + print 'No need to update' + + response = super(CloudSiteViewSet, self).retrieve(request) + # Wrap data in a dict so that it can display in template. + if type(request.accepted_renderer) is TemplateHTMLRenderer: + # Single result put in list to work with same HTML template. + response.data = {'sites': [response.data], 'last_fetched': last_fetched} + return response From 71268e26d98771b8ddf8d8ad874217207603dba1 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 30 Apr 2019 16:57:28 +0100 Subject: [PATCH 004/183] Add placeholder Nagios style result to site output --- monitoring/publishing/views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 5aaaf89..dd5a613 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -49,4 +49,8 @@ def retrieve(self, request, pk=None): if type(request.accepted_renderer) is TemplateHTMLRenderer: # Single result put in list to work with same HTML template. response.data = {'sites': [response.data], 'last_fetched': last_fetched} + + response.data['returncode'] = 3 + response.data['stdout'] = "UNKNOWN" + return response From 13415a830d27b53f9464dc593668e5a0e1de6251 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 1 May 2019 13:48:42 +0100 Subject: [PATCH 005/183] Add VMsInLastUpdate to cloud output --- monitoring/publishing/models.py | 8 ++++++-- monitoring/publishing/serializers.py | 2 +- monitoring/publishing/templates/cloudsites.html | 3 ++- monitoring/publishing/views.py | 10 ++++++---- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py index ba65c9a..5821194 100644 --- a/monitoring/publishing/models.py +++ b/monitoring/publishing/models.py @@ -7,6 +7,7 @@ class CloudSite(models.Model): fetched = models.DateTimeField(auto_now=True) name = models.CharField(max_length=255, primary_key=True) + vms = models.IntegerField(default=0) script = models.CharField(max_length=255) updated = models.DateTimeField() @@ -16,6 +17,7 @@ class Meta: class VAnonCloudRecord(models.Model): SiteName = models.CharField(max_length=255, primary_key=True) + VMs = models.IntegerField() CloudType = models.CharField(max_length=255) UpdateTime = models.DateTimeField() @@ -24,6 +26,8 @@ class Meta: db_table = 'vanoncloudrecords' def __str__(self): - return '%s, running "%s", updated at %s' % (self.SiteName, + return '%s running "%s" updated at %s with %s records' % ( + self.SiteName, self.CloudType, - self.UpdateTime) + self.UpdateTime, + self.VMs) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 40309bb..07e14dd 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -5,4 +5,4 @@ class CloudSiteSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = CloudSite - fields = ('url', 'name', 'script', 'updated') + fields = ('url', 'name', 'vms', 'script', 'updated') diff --git a/monitoring/publishing/templates/cloudsites.html b/monitoring/publishing/templates/cloudsites.html index e146485..b0b58cf 100644 --- a/monitoring/publishing/templates/cloudsites.html +++ b/monitoring/publishing/templates/cloudsites.html @@ -7,11 +7,12 @@

Sites publishing cloud accounting records from 2018-06-19 onwards

Page last updated: {{ last_fetched|date:"c" }}

- + {% for site in sites %} + diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index dd5a613..c19361d 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -11,6 +11,7 @@ from models import CloudSite, VAnonCloudRecord from serializers import CloudSiteSerializer + class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet): queryset = CloudSite.objects.all() serializer_class = CloudSiteSerializer @@ -21,9 +22,10 @@ def list(self, request): print last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20) if last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): print 'Out of date' - fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID), CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") + fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID) as VMs, CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") + print fetchset for f in fetchset: - CloudSite.objects.update_or_create(defaults={'script': f.CloudType, 'updated': f.UpdateTime}, name=f.SiteName) + CloudSite.objects.update_or_create(defaults={'vms': f.VMs, 'script': f.CloudType, 'updated': f.UpdateTime}, name=f.SiteName) else: print 'No need to update' @@ -38,9 +40,9 @@ def retrieve(self, request, pk=None): print last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20) if last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): print 'Out of date' - fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID), CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") + fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID) as VMs, CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") for f in fetchset: - CloudSite.objects.update_or_create(defaults={'script': f.CloudType, 'updated': f.UpdateTime}, name=f.SiteName) + CloudSite.objects.update_or_create(defaults={'vms': f.VMs, 'script': f.CloudType, 'updated': f.UpdateTime}, name=f.SiteName) else: print 'No need to update' From 670853770dcb0f5934eb95ae209d60a7d3441e52 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 1 May 2019 13:49:10 +0100 Subject: [PATCH 006/183] Change schema that grid db setting uses --- monitoring/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/settings.py b/monitoring/settings.py index 649ae2d..7a243d9 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -93,7 +93,7 @@ 'ENGINE': 'django.db.backends.mysql', 'HOST': 'localhost', 'PORT': '3306', - 'NAME': 'django_test', + 'NAME': 'django_test_grid', 'USER': 'root', }, 'cloud': { From 0ac742e587f7091f36c6bbe91c14200f60df0b90 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 1 May 2019 14:08:30 +0100 Subject: [PATCH 007/183] Clean up --- monitoring/publishing/views.py | 1 - monitoring/settings.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index c19361d..051d6ff 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -23,7 +23,6 @@ def list(self, request): if last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): print 'Out of date' fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID) as VMs, CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") - print fetchset for f in fetchset: CloudSite.objects.update_or_create(defaults={'vms': f.VMs, 'script': f.CloudType, 'updated': f.UpdateTime}, name=f.SiteName) else: diff --git a/monitoring/settings.py b/monitoring/settings.py index 7a243d9..9c123cc 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -38,7 +38,7 @@ 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework', - 'monitoring.publishing' + 'monitoring.publishing', ] REST_FRAMEWORK = { From c256acf4a660a6b79b70ae287b033e39e138a7ab Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 1 May 2019 14:15:33 +0100 Subject: [PATCH 008/183] Initial work on grid monitoring --- monitoring/publishing/models.py | 6 ++++++ monitoring/publishing/serializers.py | 7 +++++++ monitoring/publishing/urls.py | 1 + monitoring/publishing/views.py | 10 ++++++++-- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py index 5821194..0a73d8d 100644 --- a/monitoring/publishing/models.py +++ b/monitoring/publishing/models.py @@ -4,6 +4,12 @@ from django.db import models +class GridSite(models.Model): + fetched = models.DateTimeField(auto_now=True) + name = models.CharField(max_length=255, primary_key=True) + updated = models.DateTimeField() + + class CloudSite(models.Model): fetched = models.DateTimeField(auto_now=True) name = models.CharField(max_length=255, primary_key=True) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 07e14dd..38eb3ff 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -2,6 +2,13 @@ from models import CloudSite + +class GridSiteSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = CloudSite + fields = ('url', 'name', 'updated') + + class CloudSiteSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = CloudSite diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py index c45044e..154fad4 100644 --- a/monitoring/publishing/urls.py +++ b/monitoring/publishing/urls.py @@ -6,6 +6,7 @@ router = routers.SimpleRouter() router.register(r'cloud', views.CloudSiteViewSet) +router.register(r'grid', views.GridSiteViewSet) urlpatterns = [ diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 051d6ff..2ef951b 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -8,8 +8,14 @@ from rest_framework import viewsets from rest_framework.renderers import TemplateHTMLRenderer -from models import CloudSite, VAnonCloudRecord -from serializers import CloudSiteSerializer +from models import GridSite, CloudSite, VAnonCloudRecord +from serializers import GridSiteSerializer, CloudSiteSerializer + + +class GridSiteViewSet(viewsets.ReadOnlyModelViewSet): + queryset = GridSite.objects.all() + serializer_class = GridSiteSerializer + template_name = 'gridsites.html' class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet): From 6a4c786178d11933ccf6cd2a434a3fd4fdc24616 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 1 May 2019 14:44:20 +0100 Subject: [PATCH 009/183] Add basic grid data retrieval --- monitoring/publishing/models.py | 9 +++++++++ monitoring/publishing/views.py | 26 +++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py index 0a73d8d..1624b5a 100644 --- a/monitoring/publishing/models.py +++ b/monitoring/publishing/models.py @@ -10,6 +10,15 @@ class GridSite(models.Model): updated = models.DateTimeField() +class VSuperSummaries(models.Model): + Site = models.CharField(max_length=255, primary_key=True) + LatestPublish = models.DateTimeField() + + class Meta: + managed = False + db_table = 'VSuperSummaries' + + class CloudSite(models.Model): fetched = models.DateTimeField(auto_now=True) name = models.CharField(max_length=255, primary_key=True) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 2ef951b..93bdb03 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -8,7 +8,7 @@ from rest_framework import viewsets from rest_framework.renderers import TemplateHTMLRenderer -from models import GridSite, CloudSite, VAnonCloudRecord +from models import GridSite, VSuperSummaries, CloudSite, VAnonCloudRecord from serializers import GridSiteSerializer, CloudSiteSerializer @@ -17,6 +17,30 @@ class GridSiteViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = GridSiteSerializer template_name = 'gridsites.html' + def retrieve(self, request, pk=None): + last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max'] + # If there's no data then last_fetched is None. + if last_fetched is not None: + print last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20) + if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): + print 'Out of date' + fetchset = VSuperSummaries.objects.using('grid').raw("SELECT Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries WHERE Year=2019 GROUP BY 1;") + for f in fetchset: + GridSite.objects.update_or_create(defaults={'updated': f.LatestPublish}, name=f.Site) + else: + print 'No need to update' + + response = super(GridSiteViewSet, self).retrieve(request) + # Wrap data in a dict so that it can display in template. + if type(request.accepted_renderer) is TemplateHTMLRenderer: + # Single result put in list to work with same HTML template. + response.data = {'sites': [response.data], 'last_fetched': last_fetched} + + response.data['returncode'] = 3 + response.data['stdout'] = "UNKNOWN" + + return response + class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet): queryset = CloudSite.objects.all() From c2a0a23f8f90d0e8b431a44eb2e44944f4c40e11 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 1 May 2019 17:34:40 +0100 Subject: [PATCH 010/183] Add HTML template for grid --- .../publishing/templates/gridsites.html | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 monitoring/publishing/templates/gridsites.html diff --git a/monitoring/publishing/templates/gridsites.html b/monitoring/publishing/templates/gridsites.html new file mode 100644 index 0000000..3150b3c --- /dev/null +++ b/monitoring/publishing/templates/gridsites.html @@ -0,0 +1,35 @@ + + + + + +APEL Publication Summary : {{ sites.0.name }} + + + + +

APEL Publication Test

+
    +
  • Displays the last time the site published accounting data to the GOC. +
  • A warning / error is raised if the site has not published accounting data for 7 / 31 days, if a site has not published data for 31 days, which usually signifies a problem with APEL or RGMA services. +
  • Information about APEL APEL Wiki +
  • Contact: apel-admins [at] stfc.ac.uk +
  • lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}
+ +
SiteCloudTypeLastUpdatedSiteVMsInLastUpdateCloudTypeLastUpdated
{{ site.name }}{{ site.vms }} {{ site.script }} {{ site.updated }}
+ + + + + + + + + + + + + +
{{ sites.0.name }}
ExecutingSiteMeasurementDateMeasurementTimePublication
Status
{{ sites.0.name }}{{ sites.0.updated }}{{ sites.0.updated }}OK [ last published 1 days ago: 2019-04-30 ]
+ + \ No newline at end of file From 71d66a97c3a05fa5c906a5beea2587044ae8591d Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 1 May 2019 17:35:40 +0100 Subject: [PATCH 011/183] Tweak datetime filter --- monitoring/publishing/templates/cloudsites.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/publishing/templates/cloudsites.html b/monitoring/publishing/templates/cloudsites.html index b0b58cf..7356e02 100644 --- a/monitoring/publishing/templates/cloudsites.html +++ b/monitoring/publishing/templates/cloudsites.html @@ -4,7 +4,7 @@

Sites publishing cloud accounting records from 2018-06-19 onwards

-

Page last updated: {{ last_fetched|date:"c" }}

+

Page last updated: {{ last_fetched|date:"Y-m-d H:i:s.u" }}

From bad274e65999033a4eb32d08fa3d0579cebdf06d Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 11 Jun 2019 16:33:35 +0100 Subject: [PATCH 012/183] Update Django version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6211931..5e66705 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==1.11.20 +Django==1.11.21 djangorestframework==3.9.3 mysqlclient==1.3.4 pytz==2019.1 From ea841a37191a14465efc2d6b9e79710a959266ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jul 2019 21:19:26 +0000 Subject: [PATCH 013/183] Bump django from 1.11.21 to 1.11.22 Bumps [django](https://github.com/django/django) from 1.11.21 to 1.11.22. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/1.11.21...1.11.22) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5e66705..a364e0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==1.11.21 +Django==1.11.22 djangorestframework==3.9.3 mysqlclient==1.3.4 pytz==2019.1 From 31cf23de4a71bbfe7b9c8d1101f753fdc3f7466b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2019 02:14:29 +0000 Subject: [PATCH 014/183] Bump django from 1.11.22 to 1.11.23 Bumps [django](https://github.com/django/django) from 1.11.22 to 1.11.23. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/1.11.22...1.11.23) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a364e0c..ccf1cf4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==1.11.22 +Django==1.11.23 djangorestframework==3.9.3 mysqlclient==1.3.4 pytz==2019.1 From 2e8a5b91386e06a92fc734ca73ff59f6e422934b Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 22 Nov 2019 11:33:38 +0000 Subject: [PATCH 015/183] Update requirements --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index ccf1cf4..3d8bf06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==1.11.23 -djangorestframework==3.9.3 +Django==1.11.26 +djangorestframework==3.9.4 mysqlclient==1.3.4 -pytz==2019.1 +pytz==2019.3 From 77c200f6e5dc638fd2699860d7c21d67e1e86f30 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 22 Nov 2019 13:03:56 +0000 Subject: [PATCH 016/183] Update mysqlclient requirement --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d8bf06..00f7541 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ Django==1.11.26 djangorestframework==3.9.4 -mysqlclient==1.3.4 +mysqlclient==1.3.9 pytz==2019.3 From 18b838fa4284be8900e15efc99ea87da4e51f9e9 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 22 Nov 2019 16:44:05 +0000 Subject: [PATCH 017/183] Fix formatting of datetime in cloud HTML --- monitoring/publishing/serializers.py | 5 +++++ monitoring/publishing/templates/cloudsites.html | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 38eb3ff..7676f48 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -10,6 +10,11 @@ class Meta: class CloudSiteSerializer(serializers.HyperlinkedModelSerializer): + # Override default format with None so that Python datetime is used as + # ouput format. Encoding will be determined by the renderer and can be + # formatted by a template filter. + updated = serializers.DateTimeField(format=None) + class Meta: model = CloudSite fields = ('url', 'name', 'vms', 'script', 'updated') diff --git a/monitoring/publishing/templates/cloudsites.html b/monitoring/publishing/templates/cloudsites.html index 7356e02..a10ae1c 100644 --- a/monitoring/publishing/templates/cloudsites.html +++ b/monitoring/publishing/templates/cloudsites.html @@ -14,7 +14,7 @@

Sites publishing cloud accounting records from 2018-06-19 onwards

 - + {% endfor %}
SiteVMsInLastUpdateCloudTypeLastUpdated{{ site.name }} {{ site.vms }} {{ site.script }}{{ site.updated }}{{ site.updated|date:"Y-m-d H:i:s" }}
From 9f9dc46882638cbb4279e61afc5bc07eefb58b89 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 22 Nov 2019 16:45:28 +0000 Subject: [PATCH 018/183] Correct model used for grid serializer --- monitoring/publishing/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 7676f48..6e5ad2c 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -1,11 +1,11 @@ from rest_framework import serializers -from models import CloudSite +from models import CloudSite, GridSite class GridSiteSerializer(serializers.HyperlinkedModelSerializer): class Meta: - model = CloudSite + model = GridSite fields = ('url', 'name', 'updated') From 4cd87e95bf9039cc4edae2f26df35d9f83f23f0e Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 22 Nov 2019 17:00:21 +0000 Subject: [PATCH 019/183] Add header to cloud HTML --- monitoring/publishing/templates/cloudsites.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/monitoring/publishing/templates/cloudsites.html b/monitoring/publishing/templates/cloudsites.html index a10ae1c..e9c2bb7 100644 --- a/monitoring/publishing/templates/cloudsites.html +++ b/monitoring/publishing/templates/cloudsites.html @@ -1,3 +1,8 @@ + + + + + Sites publishing cloud accounting records From fb4d50fbc0fd1d27b579862095de5917c8c521b3 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 22 Nov 2019 17:13:46 +0000 Subject: [PATCH 020/183] Fix datetime fields for grid --- monitoring/publishing/serializers.py | 5 +++++ monitoring/publishing/templates/gridsites.html | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 6e5ad2c..3e6eb06 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -4,6 +4,11 @@ class GridSiteSerializer(serializers.HyperlinkedModelSerializer): + # Override default format with None so that Python datetime is used as + # ouput format. Encoding will be determined by the renderer and can be + # formatted by a template filter. + updated = serializers.DateTimeField(format=None) + class Meta: model = GridSite fields = ('url', 'name', 'updated') diff --git a/monitoring/publishing/templates/gridsites.html b/monitoring/publishing/templates/gridsites.html index 3150b3c..696cbcd 100644 --- a/monitoring/publishing/templates/gridsites.html +++ b/monitoring/publishing/templates/gridsites.html @@ -26,8 +26,8 @@

APEL Publication Test

{{ sites.0.name }} - {{ sites.0.updated }} - {{ sites.0.updated }} + {{ sites.0.updated|date:"Y-m-d" }} + {{ sites.0.updated|date:"G:i:s" }} OK [ last published 1 days ago: 2019-04-30 ] From 0879c1368d71d260210b48aefef263a68d781e70 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 22 Nov 2019 17:36:07 +0000 Subject: [PATCH 021/183] Change what's shown in grid status column --- monitoring/publishing/templates/gridsites.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/publishing/templates/gridsites.html b/monitoring/publishing/templates/gridsites.html index 696cbcd..b06f394 100644 --- a/monitoring/publishing/templates/gridsites.html +++ b/monitoring/publishing/templates/gridsites.html @@ -28,7 +28,7 @@

APEL Publication Test

{{ sites.0.name }} {{ sites.0.updated|date:"Y-m-d" }} {{ sites.0.updated|date:"G:i:s" }} - OK [ last published 1 days ago: 2019-04-30 ] + {{ stdout }} From 63c08d2545ed042467cdb1445dcc627d62fb8e84 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 22 Nov 2019 18:14:43 +0000 Subject: [PATCH 022/183] Change which field is used for measurement --- monitoring/publishing/templates/gridsites.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/templates/gridsites.html b/monitoring/publishing/templates/gridsites.html index b06f394..ad5d5a5 100644 --- a/monitoring/publishing/templates/gridsites.html +++ b/monitoring/publishing/templates/gridsites.html @@ -26,8 +26,8 @@

APEL Publication Test

{{ sites.0.name }} - {{ sites.0.updated|date:"Y-m-d" }} - {{ sites.0.updated|date:"G:i:s" }} + {{ last_fetched|date:"Y-m-d" }} + {{ last_fetched|date:"G:i:s" }} {{ stdout }} From c7fe6cae9faaf9d71278b54962173fedbdf2dbf7 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 22 Nov 2019 18:15:01 +0000 Subject: [PATCH 023/183] Add basic checking of publishing time --- monitoring/publishing/views.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 93bdb03..2d26a94 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -31,13 +31,23 @@ def retrieve(self, request, pk=None): print 'No need to update' response = super(GridSiteViewSet, self).retrieve(request) + date = response.data['updated'].replace(tzinfo=None) + # Wrap data in a dict so that it can display in template. if type(request.accepted_renderer) is TemplateHTMLRenderer: # Single result put in list to work with same HTML template. response.data = {'sites': [response.data], 'last_fetched': last_fetched} - response.data['returncode'] = 3 - response.data['stdout'] = "UNKNOWN" + diff = datetime.today() - date + if diff <= timedelta(days=7): + response.data['returncode'] = 0 + response.data['stdout'] = "OK [ last published %s days ago: %s ]" % (diff.days, date.strftime("%Y-%m-%d")) + elif diff > timedelta(days=7): + response.data['returncode'] = 1 + response.data['stdout'] = "WARNING [ last published %s days ago: %s ]" % (diff.days, date.strftime("%Y-%m-%d")) + else: + response.data['returncode'] = 3 + response.data['stdout'] = "UNKNOWN" return response From 24cc4dd24269fedcee9ae804265f66a217e8aeb6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Feb 2020 09:28:39 +0000 Subject: [PATCH 024/183] Bump django from 1.11.26 to 1.11.28 Bumps [django](https://github.com/django/django) from 1.11.26 to 1.11.28. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/1.11.26...1.11.28) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 00f7541..8645be0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==1.11.26 +Django==1.11.28 djangorestframework==3.9.4 mysqlclient==1.3.9 pytz==2019.3 From 3719da0f1f0dae48202e9f0827feac432924ebd6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Jun 2020 18:46:14 +0000 Subject: [PATCH 025/183] Bump django from 1.11.28 to 1.11.29 Bumps [django](https://github.com/django/django) from 1.11.28 to 1.11.29. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/1.11.28...1.11.29) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8645be0..c565648 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==1.11.28 +Django==1.11.29 djangorestframework==3.9.4 mysqlclient==1.3.9 pytz==2019.3 From c6949f3255403dc1e4dc82130cba8794092730fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Mar 2021 22:05:52 +0000 Subject: [PATCH 026/183] Bump djangorestframework from 3.9.4 to 3.11.2 Bumps [djangorestframework](https://github.com/encode/django-rest-framework) from 3.9.4 to 3.11.2. - [Release notes](https://github.com/encode/django-rest-framework/releases) - [Commits](https://github.com/encode/django-rest-framework/compare/3.9.4...3.11.2) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c565648..d21814a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ Django==1.11.29 -djangorestframework==3.9.4 +djangorestframework==3.11.2 mysqlclient==1.3.9 pytz==2019.3 From c15c99d3715389d2fff308e998b760e5b4c37428 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Jun 2021 20:41:40 +0000 Subject: [PATCH 027/183] Bump django from 1.11.29 to 2.2.24 Bumps [django](https://github.com/django/django) from 1.11.29 to 2.2.24. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/1.11.29...2.2.24) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d21814a..ac2019e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==1.11.29 +Django==2.2.24 djangorestframework==3.11.2 mysqlclient==1.3.9 pytz==2019.3 From 184a691e517128ea868212bbe98780b14fa7639a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Feb 2022 08:18:45 +0000 Subject: [PATCH 028/183] Bump django from 2.2.24 to 2.2.27 Bumps [django](https://github.com/django/django) from 2.2.24 to 2.2.27. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/2.2.24...2.2.27) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ac2019e..837bf17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==2.2.24 +Django==2.2.27 djangorestframework==3.11.2 mysqlclient==1.3.9 pytz==2019.3 From 4093e61f9d173ec1513990c90d8146792595ddf9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Apr 2022 09:26:17 +0100 Subject: [PATCH 029/183] Bump django from 2.2.27 to 2.2.28 (#11) Bumps [django](https://github.com/django/django) from 2.2.27 to 2.2.28. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/2.2.27...2.2.28) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 837bf17..54be296 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==2.2.27 +Django==2.2.28 djangorestframework==3.11.2 mysqlclient==1.3.9 pytz==2019.3 From b6d23f1d7141af674c754619041a43ce4c2d9533 Mon Sep 17 00:00:00 2001 From: Letizia Protopapa Date: Wed, 22 Feb 2023 16:13:30 +0000 Subject: [PATCH 030/183] Add file explaining how to get the app working on an aq machine. --- docs/setting_up_VM_with_app.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 docs/setting_up_VM_with_app.md diff --git a/docs/setting_up_VM_with_app.md b/docs/setting_up_VM_with_app.md new file mode 100644 index 0000000..be7f3f3 --- /dev/null +++ b/docs/setting_up_VM_with_app.md @@ -0,0 +1,25 @@ +## Steps to follow to get a VM with monitoring app running on Apache + +To get a 'prototype' of the monitoring app running with Apache, follow these steps: +- create a cloud VM of the type: scientific-linux-7-aq +- continue by selecting sandbox 'testing_personality_2', archetype 'ral-tier1', personality 'apel-data-validation-test' + +Allow 15 minutes after the machine is created, then follow these steps from within the machine: + +- yum remove git && quattor-fetch && quattor-configure --all +- cd /usr/share/DJANGO_MONITORING_APP/monitoring +- modify the file settings.py, specifically the dict called DATABASES, to include the right credentials so as to access the right dbs +- cd .. +- source venv/bin/activate +- systemctl restart httpd +- sudo chown apache . + +At this point the app should be working, so just get the ip address by writing "hostname -I" within the machine and the app should be already running at that address. + + +## What to do if the app seems to stop working +If the VM is shut down, next time we try to open the app, Apache might give the error message "Unable to open the database file". +If this happens, just follow these steps on the machine: +- cd /usr/share/DJANGO_MONITORING_APP +- source venv/bin/activate +- sudo chown apache . From 282dc43c7a46ac334d2f18592d0ec8632cbb2cde Mon Sep 17 00:00:00 2001 From: Letizia Protopapa Date: Wed, 22 Feb 2023 16:16:12 +0000 Subject: [PATCH 031/183] Add file explaining how to access pages of the monitoring app. --- docs/what_pages_can_be_accessed.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docs/what_pages_can_be_accessed.md diff --git a/docs/what_pages_can_be_accessed.md b/docs/what_pages_can_be_accessed.md new file mode 100644 index 0000000..0e211cc --- /dev/null +++ b/docs/what_pages_can_be_accessed.md @@ -0,0 +1,22 @@ +## Pages that can be accessed through the app + +The following urls are the ones that can be accessed without passing any parameter: +- http://ip-address/publishing/cloud/ +- http://ip-address/publishing/gridsync/ + +These pages show info for a number of sites, so do not require a site name to be specified within the url. + +The url http://ip-address/publishing/grid/ , instead, should be used together with the name of the site we are looking for. +For example: http://ip-address/publishing/grid/BelGrid-UCL/ +It is not supposed to be used without passing the name of the site. + +The url http://ip-address/publishing/gridsync/ shows a sync table, and it's probably the most important bit of the personality 'apel-data-validation'. +This table contains data related to many sites, specifically number of jobs being published vs in the db, and this number is shown for every (available) month of each site. + +Clicking on any name in the first column (containing site names) allows to access a similar table which only shows the data relative to that site. +This more specific table is such that the first columns shows the months for which we have data (for that site). +Clicking on the month allows to open another table that shows data for that month and site only, divided by submithost. + +The pages accessed through the links can of course be accessed by typing directly the url. For example, if we want data related to the site 'CSCS-LCG2' and month '2013-11', we would type : +http://ip-address/publishing/gridsync/CSCS-LCG2/2013-11/ +However, in this case if there is no data for the month we are looking for, we would get an error. From d94b4dc9c027ed982ad6d80ae64207a25d0166ad Mon Sep 17 00:00:00 2001 From: Letizia Protopapa Date: Wed, 22 Feb 2023 16:33:52 +0000 Subject: [PATCH 032/183] Add file listing packages installed by Quattor. --- docs/what_gets_installed.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docs/what_gets_installed.md diff --git a/docs/what_gets_installed.md b/docs/what_gets_installed.md new file mode 100644 index 0000000..c096610 --- /dev/null +++ b/docs/what_gets_installed.md @@ -0,0 +1,23 @@ +For Django to work with apache, it is common to have a venv within the app, where django and djangorestframework get installed. Other packages needed for the app to work are installed by Aquilon outside the venv. + +## Packages installed by Aquilon outside the venv +Following the config file that Aquilon uses, the following are the packages installed, in this order: +- httpd +- python3-mod_wsgi (for apache to work with django) +- python3-devel +- gcc (needed for dependencies) +- mariadb. + + +## Packages installed within the venv +Within venv, the following are installed through pip: +- djangorestframework (3.11.2) +- pymysql (needed for mariadb to work) +- pandas (needed by the app) +- django (2.1.*). + +Note that when the version of the packages is specified, the app would not work with a different version (due to dependencies conflicts). +Also, as long as mariadb is installed (both client and server), it is not necessary to install mysqlclient (at least when the OS is Scientific Linux). + +Is is also important to note that different types of OS require different packages to be installed. +The above are the packages that allow the app to work on a scientific linux 7 machine. A Centos machine would require slightly different packages. From 7422c4be36f34827d202fcc0d1c9ae315810118a Mon Sep 17 00:00:00 2001 From: Letizia Protopapa Date: Thu, 23 Feb 2023 13:58:54 +0000 Subject: [PATCH 033/183] Add reminder about modifying security groups. --- docs/setting_up_VM_with_app.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/setting_up_VM_with_app.md b/docs/setting_up_VM_with_app.md index be7f3f3..6af00c0 100644 --- a/docs/setting_up_VM_with_app.md +++ b/docs/setting_up_VM_with_app.md @@ -2,10 +2,10 @@ To get a 'prototype' of the monitoring app running with Apache, follow these steps: - create a cloud VM of the type: scientific-linux-7-aq -- continue by selecting sandbox 'testing_personality_2', archetype 'ral-tier1', personality 'apel-data-validation-test' - -Allow 15 minutes after the machine is created, then follow these steps from within the machine: +- continue by selecting sandbox 'testing_personality_2', archetype 'ral-tier1', personality 'apel-data-validation-test' +Allow 15 minutes after the machine is created, then remember to edit security groups from OpenStack to allow Apache to work. +Then follow these steps from within the machine: - yum remove git && quattor-fetch && quattor-configure --all - cd /usr/share/DJANGO_MONITORING_APP/monitoring - modify the file settings.py, specifically the dict called DATABASES, to include the right credentials so as to access the right dbs @@ -14,7 +14,7 @@ Allow 15 minutes after the machine is created, then follow these steps from with - systemctl restart httpd - sudo chown apache . -At this point the app should be working, so just get the ip address by writing "hostname -I" within the machine and the app should be already running at that address. +At this point the app should be working, so just get the ip address by writing "hostname -I" within the machine and the app should be already running at that address. ## What to do if the app seems to stop working From be3e742f73a1b0a5fbb6f862f037c8b69073daa5 Mon Sep 17 00:00:00 2001 From: Letizia Protopapa Date: Thu, 23 Feb 2023 13:59:59 +0000 Subject: [PATCH 034/183] Improve clarity of content. --- docs/setting_up_VM_with_app.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/setting_up_VM_with_app.md b/docs/setting_up_VM_with_app.md index 6af00c0..d6d1d12 100644 --- a/docs/setting_up_VM_with_app.md +++ b/docs/setting_up_VM_with_app.md @@ -8,7 +8,7 @@ Allow 15 minutes after the machine is created, then remember to edit security gr Then follow these steps from within the machine: - yum remove git && quattor-fetch && quattor-configure --all - cd /usr/share/DJANGO_MONITORING_APP/monitoring -- modify the file settings.py, specifically the dict called DATABASES, to include the right credentials so as to access the right dbs +- modify the file settings.py, specifically the dict called DATABASES, to include the right credentials and database names - cd .. - source venv/bin/activate - systemctl restart httpd @@ -17,9 +17,12 @@ Then follow these steps from within the machine: At this point the app should be working, so just get the ip address by writing "hostname -I" within the machine and the app should be already running at that address. -## What to do if the app seems to stop working +## What to do if the app seems to stop working after closing the VM If the VM is shut down, next time we try to open the app, Apache might give the error message "Unable to open the database file". If this happens, just follow these steps on the machine: -- cd /usr/share/DJANGO_MONITORING_APP -- source venv/bin/activate -- sudo chown apache . +1. cd /usr/share/DJANGO_MONITORING_APP +2. source venv/bin/activate +3. sudo chown apache . + +Note that step 2 is necessary for step 3 to work and the error message to disappear. + From ef2932d0004dc8ff38aaa51688005048241df909 Mon Sep 17 00:00:00 2001 From: Letizia Protopapa Date: Thu, 23 Feb 2023 14:31:40 +0000 Subject: [PATCH 035/183] Remove whitespaces and unnecessary blank lines. --- docs/setting_up_VM_with_app.md | 1 - docs/what_pages_can_be_accessed.md | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/setting_up_VM_with_app.md b/docs/setting_up_VM_with_app.md index d6d1d12..e8346f8 100644 --- a/docs/setting_up_VM_with_app.md +++ b/docs/setting_up_VM_with_app.md @@ -25,4 +25,3 @@ If this happens, just follow these steps on the machine: 3. sudo chown apache . Note that step 2 is necessary for step 3 to work and the error message to disappear. - diff --git a/docs/what_pages_can_be_accessed.md b/docs/what_pages_can_be_accessed.md index 0e211cc..6f83fb7 100644 --- a/docs/what_pages_can_be_accessed.md +++ b/docs/what_pages_can_be_accessed.md @@ -6,7 +6,7 @@ The following urls are the ones that can be accessed without passing any paramet These pages show info for a number of sites, so do not require a site name to be specified within the url. -The url http://ip-address/publishing/grid/ , instead, should be used together with the name of the site we are looking for. +The url http://ip-address/publishing/grid/ , instead, should be used together with the name of the site we are looking for. For example: http://ip-address/publishing/grid/BelGrid-UCL/ It is not supposed to be used without passing the name of the site. @@ -14,9 +14,9 @@ The url http://ip-address/publishing/gridsync/ shows a sync table, and it's prob This table contains data related to many sites, specifically number of jobs being published vs in the db, and this number is shown for every (available) month of each site. Clicking on any name in the first column (containing site names) allows to access a similar table which only shows the data relative to that site. -This more specific table is such that the first columns shows the months for which we have data (for that site). +This more specific table is such that the first columns shows the months for which we have data (for that site). Clicking on the month allows to open another table that shows data for that month and site only, divided by submithost. The pages accessed through the links can of course be accessed by typing directly the url. For example, if we want data related to the site 'CSCS-LCG2' and month '2013-11', we would type : -http://ip-address/publishing/gridsync/CSCS-LCG2/2013-11/ +http://ip-address/publishing/gridsync/CSCS-LCG2/2013-11/ However, in this case if there is no data for the month we are looking for, we would get an error. From b751833ede88dd9514144d596f022c873a7abdd3 Mon Sep 17 00:00:00 2001 From: Adrian Coveney <4836233+tofu-rocketry@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:11:54 +0000 Subject: [PATCH 036/183] Apply suggestions from code review Co-authored-by: gregcorbett --- docs/setting_up_VM_with_app.md | 2 +- docs/what_gets_installed.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/setting_up_VM_with_app.md b/docs/setting_up_VM_with_app.md index e8346f8..254bf0f 100644 --- a/docs/setting_up_VM_with_app.md +++ b/docs/setting_up_VM_with_app.md @@ -6,7 +6,7 @@ To get a 'prototype' of the monitoring app running with Apache, follow these ste Allow 15 minutes after the machine is created, then remember to edit security groups from OpenStack to allow Apache to work. Then follow these steps from within the machine: -- yum remove git && quattor-fetch && quattor-configure --all +- quattor-fetch && quattor-configure --all - cd /usr/share/DJANGO_MONITORING_APP/monitoring - modify the file settings.py, specifically the dict called DATABASES, to include the right credentials and database names - cd .. diff --git a/docs/what_gets_installed.md b/docs/what_gets_installed.md index c096610..0d43cce 100644 --- a/docs/what_gets_installed.md +++ b/docs/what_gets_installed.md @@ -1,7 +1,7 @@ For Django to work with apache, it is common to have a venv within the app, where django and djangorestframework get installed. Other packages needed for the app to work are installed by Aquilon outside the venv. ## Packages installed by Aquilon outside the venv -Following the config file that Aquilon uses, the following are the packages installed, in this order: +Following the config file that Aquilon uses, the following are the packages installed: - httpd - python3-mod_wsgi (for apache to work with django) - python3-devel From 09a0b8cc5b9e2397ff02992877d2ca9f69e00468 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 6 Jan 2023 11:39:55 +0000 Subject: [PATCH 037/183] Ensure Python3 compatibility. --- monitoring/publishing/views.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 2d26a94..e0e1d5d 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -21,14 +21,14 @@ def retrieve(self, request, pk=None): last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max'] # If there's no data then last_fetched is None. if last_fetched is not None: - print last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20) + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): - print 'Out of date' + print('Out of date') fetchset = VSuperSummaries.objects.using('grid').raw("SELECT Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries WHERE Year=2019 GROUP BY 1;") for f in fetchset: GridSite.objects.update_or_create(defaults={'updated': f.LatestPublish}, name=f.Site) else: - print 'No need to update' + print('No need to update') response = super(GridSiteViewSet, self).retrieve(request) date = response.data['updated'].replace(tzinfo=None) @@ -59,14 +59,14 @@ class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet): def list(self, request): last_fetched = CloudSite.objects.aggregate(Max('fetched'))['fetched__max'] - print last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20) + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) if last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): - print 'Out of date' + print('Out of date') fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID) as VMs, CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") for f in fetchset: CloudSite.objects.update_or_create(defaults={'vms': f.VMs, 'script': f.CloudType, 'updated': f.UpdateTime}, name=f.SiteName) else: - print 'No need to update' + print('No need to update') response = super(CloudSiteViewSet, self).list(request) # Wrap data in a dict so that it can display in template. @@ -76,14 +76,14 @@ def list(self, request): def retrieve(self, request, pk=None): last_fetched = CloudSite.objects.aggregate(Max('fetched'))['fetched__max'] - print last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20) + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) if last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): - print 'Out of date' + print('Out of date') fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID) as VMs, CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") for f in fetchset: CloudSite.objects.update_or_create(defaults={'vms': f.VMs, 'script': f.CloudType, 'updated': f.UpdateTime}, name=f.SiteName) else: - print 'No need to update' + print('No need to update') response = super(CloudSiteViewSet, self).retrieve(request) # Wrap data in a dict so that it can display in template. From abbe8c0c472e3bcc431107873f50944fd5915150 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 6 Jan 2023 11:44:43 +0000 Subject: [PATCH 038/183] Make imports more explicit. --- monitoring/availability/urls.py | 4 +++- monitoring/publishing/serializers.py | 2 +- monitoring/publishing/urls.py | 2 +- monitoring/publishing/views.py | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/monitoring/availability/urls.py b/monitoring/availability/urls.py index 0c9a403..edf402e 100644 --- a/monitoring/availability/urls.py +++ b/monitoring/availability/urls.py @@ -1,6 +1,8 @@ from django.conf.urls import url -import views +import sys +sys.path.append('/usr/share/DJANGO_MONITORING_APP') +from monitoring.availability import views urlpatterns = [ url(r'^$', views.status), diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 3e6eb06..aef7ae8 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from models import CloudSite, GridSite +from monitoring.publishing.models import CloudSite, GridSite class GridSiteSerializer(serializers.HyperlinkedModelSerializer): diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py index 154fad4..9691e8d 100644 --- a/monitoring/publishing/urls.py +++ b/monitoring/publishing/urls.py @@ -2,7 +2,7 @@ from rest_framework import routers -import views +from monitoring.publishing import views router = routers.SimpleRouter() router.register(r'cloud', views.CloudSiteViewSet) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index e0e1d5d..f296fad 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -8,8 +8,8 @@ from rest_framework import viewsets from rest_framework.renderers import TemplateHTMLRenderer -from models import GridSite, VSuperSummaries, CloudSite, VAnonCloudRecord -from serializers import GridSiteSerializer, CloudSiteSerializer +from monitoring.publishing.models import GridSite, VSuperSummaries, CloudSite, VAnonCloudRecord +from monitoring.publishing.serializers import GridSiteSerializer, CloudSiteSerializer class GridSiteViewSet(viewsets.ReadOnlyModelViewSet): From 9bd9ec0ed53467bedc41dce8c266397d8e6be391 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 6 Jan 2023 11:45:40 +0000 Subject: [PATCH 039/183] Add allowed hosts. --- monitoring/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/settings.py b/monitoring/settings.py index 9c123cc..bbeaadb 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -25,7 +25,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = [ '127.0.0.1', 'localhost'] # Application definition From 96e1b4a146dabb6776eeb424b1064e8b6137b28d Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 16:43:32 +0000 Subject: [PATCH 040/183] Add static root to be able to run "manage.py collectstatic". --- monitoring/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/monitoring/settings.py b/monitoring/settings.py index bbeaadb..e1da566 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -143,3 +143,4 @@ # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(BASE_DIR, "static/") \ No newline at end of file From c8a79d8a61d6668cf90cce49747bde31a3dff73d Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 6 Jan 2023 11:47:16 +0000 Subject: [PATCH 041/183] Ensure Python3 compatibility for CloudSite view. --- monitoring/publishing/views.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index f296fad..188fd67 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -59,8 +59,9 @@ class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet): def list(self, request): last_fetched = CloudSite.objects.aggregate(Max('fetched'))['fetched__max'] - print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - if last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + if last_fetched is None or (last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20))): print('Out of date') fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID) as VMs, CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") for f in fetchset: From 6681204d0f6aefcec57860d80f5724678aa08161 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 6 Jan 2023 14:16:41 +0000 Subject: [PATCH 042/183] Ensure the grid site works when retrieving multiple records. --- monitoring/publishing/views.py | 37 +++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 188fd67..25d1e38 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -17,6 +17,41 @@ class GridSiteViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = GridSiteSerializer template_name = 'gridsites.html' + def list(self, request): + last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max'] + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + if last_fetched is None or (last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20))): + fetchset = VSuperSummaries.objects.using('grid').raw("SELECT Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries WHERE Year=2019 GROUP BY 1;") + for f in fetchset: + GridSite.objects.update_or_create(defaults={'updated': f.LatestPublish}, name=f.Site) + else: + print('No need to update') + + final_response = [] + response = super(GridSiteViewSet, self).list(request) + + for single_dict in response.data: + date = single_dict.get('updated').replace(tzinfo=None) + + diff = datetime.today() - date + if diff <= timedelta(days=7): + single_dict['returncode'] = 0 + single_dict['stdout'] = "OK [ last published %s days ago: %s ]" % (diff.days, date.strftime("%Y-%m-%d")) + elif diff > timedelta(days=7): + single_dict['returncode'] = 1 + single_dict['stdout'] = "WARNING [ last published %s days ago: %s ]" % (diff.days, date.strftime("%Y-%m-%d")) + else: + single_dict['returncode'] = 3 + single_dict['stdout'] = "UNKNOWN" + final_response.append(single_dict) + + if type(request.accepted_renderer) is TemplateHTMLRenderer: + response.data = {'sites': final_response, 'last_fetched': last_fetched} + + return response + + def retrieve(self, request, pk=None): last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max'] # If there's no data then last_fetched is None. @@ -95,4 +130,4 @@ def retrieve(self, request, pk=None): response.data['returncode'] = 3 response.data['stdout'] = "UNKNOWN" - return response + return response \ No newline at end of file From e9894a92e967ead2f566f77a4a97c5e61475e65d Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 10:36:54 +0000 Subject: [PATCH 043/183] Add import for pymysql. --- monitoring/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monitoring/__init__.py b/monitoring/__init__.py index e69de29..c45523b 100644 --- a/monitoring/__init__.py +++ b/monitoring/__init__.py @@ -0,0 +1,2 @@ +import pymysql +pymysql.install_as_MySQLdb() \ No newline at end of file From 8eaf7449f8dfc145d2e893f112fdd4c968e10175 Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 10:37:59 +0000 Subject: [PATCH 044/183] Add models for the sync page. --- monitoring/publishing/models.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py index 1624b5a..e4f6ecd 100644 --- a/monitoring/publishing/models.py +++ b/monitoring/publishing/models.py @@ -13,12 +13,44 @@ class GridSite(models.Model): class VSuperSummaries(models.Model): Site = models.CharField(max_length=255, primary_key=True) LatestPublish = models.DateTimeField() + Month = models.IntegerField() + Year = models.IntegerField() + RecordStart = models.DateTimeField() + RecordEnd = models.DateTimeField() + RecordCountPublished = models.IntegerField() class Meta: managed = False db_table = 'VSuperSummaries' +class GridSiteSync(models.Model): + fetched = models.DateTimeField(auto_now=True) + SiteName = models.CharField(max_length=255) + YearMonth = models.CharField(max_length=255) + Year = models.IntegerField() + Month = models.IntegerField() + RecordStart = models.DateTimeField() + RecordEnd = models.DateTimeField() + RecordCountPublished = models.IntegerField() + RecordCountInDb = models.IntegerField() + SyncStatus = models.CharField(max_length=255) + + class Meta: + # Descending order of Year and Month to display latest data first + ordering = ('SiteName', '-Year', '-Month') + unique_together = ('SiteName', 'YearMonth') + + +class VSyncRecords(models.Model): + Site = models.CharField(max_length=255, primary_key=True) + RecordCountInDb = models.IntegerField() + + class Meta: + managed = False + db_table = 'VSyncRecords' + + class CloudSite(models.Model): fetched = models.DateTimeField(auto_now=True) name = models.CharField(max_length=255, primary_key=True) From 58417ca89d9fd9ee7c6400af906efddd1e799643 Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 10:38:49 +0000 Subject: [PATCH 045/183] Modify names to ensure consistency with the new models (GridSiteSync). --- monitoring/publishing/models.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py index e4f6ecd..f0e1eb7 100644 --- a/monitoring/publishing/models.py +++ b/monitoring/publishing/models.py @@ -6,7 +6,7 @@ class GridSite(models.Model): fetched = models.DateTimeField(auto_now=True) - name = models.CharField(max_length=255, primary_key=True) + SiteName = models.CharField(max_length=255, primary_key=True) updated = models.DateTimeField() @@ -18,7 +18,7 @@ class VSuperSummaries(models.Model): RecordStart = models.DateTimeField() RecordEnd = models.DateTimeField() RecordCountPublished = models.IntegerField() - + class Meta: managed = False db_table = 'VSuperSummaries' @@ -53,13 +53,13 @@ class Meta: class CloudSite(models.Model): fetched = models.DateTimeField(auto_now=True) - name = models.CharField(max_length=255, primary_key=True) - vms = models.IntegerField(default=0) - script = models.CharField(max_length=255) + SiteName = models.CharField(max_length=255, primary_key=True) + Vms = models.IntegerField(default=0) + Script = models.CharField(max_length=255) updated = models.DateTimeField() class Meta: - ordering = ('name',) + ordering = ('SiteName',) class VAnonCloudRecord(models.Model): @@ -78,3 +78,4 @@ def __str__(self): self.CloudType, self.UpdateTime, self.VMs) + \ No newline at end of file From 274dc73dd4f299a6dd57843d8cf25439101bab24 Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 10:40:13 +0000 Subject: [PATCH 046/183] Add serializer for gridsync site. --- monitoring/publishing/serializers.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index aef7ae8..dbd2fc2 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from monitoring.publishing.models import CloudSite, GridSite +from monitoring.publishing.models import CloudSite, GridSite, GridSiteSync class GridSiteSerializer(serializers.HyperlinkedModelSerializer): @@ -14,6 +14,22 @@ class Meta: fields = ('url', 'name', 'updated') +class GridSiteSyncSerializer(serializers.HyperlinkedModelSerializer): + # Override default format with None so that Python datetime is used as + # ouput format. Encoding will be determined by the renderer and can be + # formatted by a template filter. + + class Meta: + model = GridSiteSync + fields = ('url', 'SiteName', 'YearMonth', 'RecordStart', 'RecordEnd', 'RecordCountPublished', 'RecordCountInDb', 'SyncStatus') + + # Sitename substitutes pk + lookup_field = 'SiteName' + extra_kwargs = { + 'url': {'lookup_field': 'SiteName'} + } + + class CloudSiteSerializer(serializers.HyperlinkedModelSerializer): # Override default format with None so that Python datetime is used as # ouput format. Encoding will be determined by the renderer and can be From 40fa9725570ee617b5f1865eb8ec45ca5091bee1 Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 10:41:41 +0000 Subject: [PATCH 047/183] Modify names to ensure consistency with modified models. --- monitoring/publishing/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index dbd2fc2..3da01dc 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -11,7 +11,7 @@ class GridSiteSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = GridSite - fields = ('url', 'name', 'updated') + fields = ('url', 'SiteName', 'updated') class GridSiteSyncSerializer(serializers.HyperlinkedModelSerializer): @@ -38,4 +38,4 @@ class CloudSiteSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = CloudSite - fields = ('url', 'name', 'vms', 'script', 'updated') + fields = ('url', 'SiteName', 'Vms', 'Script', 'updated') From 14788f3cb07f277ce52c0ac0b55c15c712975e1a Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 10:45:45 +0000 Subject: [PATCH 048/183] Add view for the gridsync table, for both single site and all sites. --- monitoring/publishing/views.py | 176 ++++++++++++++++++++++++++++++++- 1 file changed, 173 insertions(+), 3 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 25d1e38..244d307 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -4,12 +4,13 @@ from datetime import datetime, timedelta from django.db.models import Max +import pandas as pd -from rest_framework import viewsets +from rest_framework import viewsets, generics from rest_framework.renderers import TemplateHTMLRenderer -from monitoring.publishing.models import GridSite, VSuperSummaries, CloudSite, VAnonCloudRecord -from monitoring.publishing.serializers import GridSiteSerializer, CloudSiteSerializer +from monitoring.publishing.models import GridSite, VSuperSummaries, CloudSite, VAnonCloudRecord, GridSiteSync, VSyncRecords +from monitoring.publishing.serializers import GridSiteSerializer, CloudSiteSerializer, GridSiteSyncSerializer class GridSiteViewSet(viewsets.ReadOnlyModelViewSet): @@ -87,6 +88,175 @@ def retrieve(self, request, pk=None): return response + +class GridSiteSyncViewSet(viewsets.ReadOnlyModelViewSet): + queryset = GridSiteSync.objects.all() + serializer_class = GridSiteSyncSerializer + lookup_field = 'SiteName' + + # When a single site is showed (retrieve function used), the template + # is different than the one used when showing a list of sites + def get_template_names(self): + if self.action == 'list': + return ['gridsync.html'] + elif self.action == 'retrieve': + return ['gridsync_singlesite.html'] + + # Combine Year and Month into one string (display purposes) + def get_year_month_string(self, year, month): + year_string = str(year) + month_string = str(month) + if len(month_string)==1: + month_string = '0'+month_string + return year_string+ '-' +month_string + + def list(self, request): + last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] + n_sites = GridSiteSync.objects.values('SiteName').distinct().count() + + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or n_sites==1: + print('Out of date') + + # The condition on EarliestEndTime and LatestEndTime is necessary to avoid error by pytz because of dates like '00-00-00' + fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE EarliestEndTime>'1900-01-01' AND LatestEndTime>'1900-01-01' GROUP BY Site, Year, Month;") + fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords GROUP BY Site, Year, Month") + + # Create empty dicts that will become dfs to be combined + Summaries_dict = {"Site":[], "Month":[], "Year":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} + SyncRecords_dict = {"Site":[], "Month":[], "Year":[],"RecordCountInDb":[]} + + # Fill the dicts with the fetched data + for row in fetchset_Summaries: + Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] + Summaries_dict["Month"] = Summaries_dict.get("Month") + [row.Month] + Summaries_dict["Year"] = Summaries_dict.get("Year") + [row.Year] + Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] + Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] + Summaries_dict["RecordEnd"] = Summaries_dict.get("RecordEnd") + [row.RecordEnd] + + for row in fetchset_SyncRecords: + SyncRecords_dict["Site"] = SyncRecords_dict.get("Site") + [row.Site] + SyncRecords_dict["Month"] = SyncRecords_dict.get("Month") + [row.Month] + SyncRecords_dict["Year"] = SyncRecords_dict.get("Year") + [row.Year] + SyncRecords_dict["RecordCountInDb"] = SyncRecords_dict.get("RecordCountInDb") + [row.RecordCountInDb] + + # Merge data from VSuperSummaries and VSyncRecords into one df + df_Summaries = pd.DataFrame.from_dict(Summaries_dict) + df_SyncRecords = pd.DataFrame.from_dict(SyncRecords_dict) + df_all = df_Summaries.merge(df_SyncRecords, left_on=['Site', 'Month', 'Year'], right_on=['Site', 'Month', 'Year'], how='inner') + fetchset = df_all.to_dict('index') + + # Delete all data if table not empty (as this function lists all sites) + GridSiteSync.objects.all().delete() + + # Determine SyncStatus based on the difference between records published and in db + for f in fetchset.values(): + rel_diff1 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountInDb")) + rel_diff2 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountPublished")) + if rel_diff1 < 0.01 or rel_diff2 < 0.01: + f['SyncStatus']='OK' + else: + f['SyncStatus']='Error' + + # Combined primary keys outside the default dict + GridSiteSync.objects.update_or_create( + defaults={ + 'RecordStart': f.get("RecordStart"), + 'RecordEnd': f.get("RecordEnd"), + 'RecordCountPublished': f.get("RecordCountPublished"), + 'RecordCountInDb': f.get("RecordCountInDb"), + 'SyncStatus': f.get("SyncStatus"), + }, + YearMonth = self.get_year_month_string(f.get("Year"), f.get("Month")), + SiteName=f.get("Site"), + Month=f.get("Month"), + Year=f.get("Year"), + ) + + else: + print('No need to update') + + response = super(GridSiteSyncViewSet, self).list(request) + response.data = {'records': response.data, 'last_fetched': last_fetched} + return response + + + + def retrieve(self, request, SiteName=None): + lookup_field = 'SiteName' + last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] + row_1 = GridSiteSync.objects.filter()[:1].get() + n_sites = GridSiteSync.objects.values('SiteName').distinct().count() + + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or n_sites > 1 or SiteName != row_1.SiteName: + print('Out of date') + + # The condition on EarliestEndTime and LatestEndTime is necessary to avoid error by pytz because of dates like '00-00-00' + fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND EarliestEndTime>'1900-01-01' AND LatestEndTime>'1900-01-01'GROUP BY Site, Month, Year;".format(SiteName)) + fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords WHERE Site='{}' GROUP BY Site, Month, Year;".format(SiteName)) + + Summaries_dict = {"Site":[], "Month":[], "Year":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} + SyncRecords_dict = {"Site":[], "Month":[], "Year":[], "RecordCountInDb":[]} + + for row in fetchset_Summaries: + Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] + Summaries_dict["Month"] = Summaries_dict.get("Month") + [row.Month] + Summaries_dict["Year"] = Summaries_dict.get("Year") + [row.Year] + Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] + Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] + Summaries_dict["RecordEnd"] = Summaries_dict.get("RecordEnd") + [row.RecordEnd] + + for row in fetchset_SyncRecords: + SyncRecords_dict["Site"] = SyncRecords_dict.get("Site") + [row.Site] + SyncRecords_dict["Month"] = SyncRecords_dict.get("Month") + [row.Month] + SyncRecords_dict["Year"] = SyncRecords_dict.get("Year") + [row.Year] + SyncRecords_dict["RecordCountInDb"] = SyncRecords_dict.get("RecordCountInDb") + [row.RecordCountInDb] + + df_Summaries = pd.DataFrame.from_dict(Summaries_dict) + df_SyncRecords = pd.DataFrame.from_dict(SyncRecords_dict) + df_all = df_Summaries.merge(df_SyncRecords, left_on=['Site', 'Month', 'Year'], right_on=['Site', 'Month', 'Year'], how='inner') + fetchset = df_all.to_dict('index') + + # Ensure we list only the data for one site + first_row = GridSiteSync.objects.first() + if first_row is not None: + if first_row.SiteName != SiteName: + GridSiteSync.objects.all().delete() + + for f in fetchset.values(): + rel_diff1 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountInDb")) + rel_diff2 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountPublished")) + if rel_diff1 <= 0.01 or rel_diff2 <= 0.01: + f['SyncStatus']='OK' + else: + f['SyncStatus']='Error' + + GridSiteSync.objects.update_or_create( + defaults={ + 'RecordStart': f.get("RecordStart"), + 'RecordEnd': f.get("RecordEnd"), + 'RecordCountPublished': f.get("RecordCountPublished"), + 'RecordCountInDb': f.get("RecordCountInDb"), + 'SyncStatus': f.get("SyncStatus"), + }, + YearMonth = self.get_year_month_string(f.get("Year"), f.get("Month")), + SiteName=f.get("Site"), + Month=f.get("Month"), + Year=f.get("Year"), + ) + + else: + print('No need to update') + + response = super(GridSiteSyncViewSet, self).list(request) + response.data = {'records': response.data, 'last_fetched': last_fetched} + return response + + class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet): queryset = CloudSite.objects.all() serializer_class = CloudSiteSerializer From 39f77d237428026ffa080021be925f7bca05c1ec Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 10:47:04 +0000 Subject: [PATCH 049/183] Update field names to ensure consistency with modified models and serialisers. --- monitoring/publishing/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 244d307..f3f456a 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -25,7 +25,7 @@ def list(self, request): if last_fetched is None or (last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20))): fetchset = VSuperSummaries.objects.using('grid').raw("SELECT Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries WHERE Year=2019 GROUP BY 1;") for f in fetchset: - GridSite.objects.update_or_create(defaults={'updated': f.LatestPublish}, name=f.Site) + GridSite.objects.update_or_create(defaults={'updated': f.LatestPublish}, SiteName=f.Site) else: print('No need to update') @@ -62,7 +62,7 @@ def retrieve(self, request, pk=None): print('Out of date') fetchset = VSuperSummaries.objects.using('grid').raw("SELECT Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries WHERE Year=2019 GROUP BY 1;") for f in fetchset: - GridSite.objects.update_or_create(defaults={'updated': f.LatestPublish}, name=f.Site) + GridSite.objects.update_or_create(defaults={'updated': f.LatestPublish}, SiteName=f.Site) else: print('No need to update') @@ -270,7 +270,7 @@ def list(self, request): print('Out of date') fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID) as VMs, CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") for f in fetchset: - CloudSite.objects.update_or_create(defaults={'vms': f.VMs, 'script': f.CloudType, 'updated': f.UpdateTime}, name=f.SiteName) + CloudSite.objects.update_or_create(defaults={'Vms': f.VMs, 'Script': f.CloudType, 'updated': f.UpdateTime}, SiteName=f.SiteName) else: print('No need to update') @@ -287,7 +287,7 @@ def retrieve(self, request, pk=None): print('Out of date') fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID) as VMs, CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") for f in fetchset: - CloudSite.objects.update_or_create(defaults={'vms': f.VMs, 'script': f.CloudType, 'updated': f.UpdateTime}, name=f.SiteName) + CloudSite.objects.update_or_create(defaults={'Vms': f.VMs, 'Script': f.CloudType, 'updated': f.UpdateTime}, SiteName=f.SiteName) else: print('No need to update') From b55b90d88d0eb1651b20f2097166239b48fb3d9a Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 10:49:08 +0000 Subject: [PATCH 050/183] Add html templates for gridsync, both all sites page and single site page. --- monitoring/publishing/templates/gridsync.html | 43 +++++++++++++++++++ .../templates/gridsync_singlesite.html | 41 ++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 monitoring/publishing/templates/gridsync.html create mode 100644 monitoring/publishing/templates/gridsync_singlesite.html diff --git a/monitoring/publishing/templates/gridsync.html b/monitoring/publishing/templates/gridsync.html new file mode 100644 index 0000000..1b43e97 --- /dev/null +++ b/monitoring/publishing/templates/gridsync.html @@ -0,0 +1,43 @@ + + + + + APEL Publication Summary + + + + +

APEL Synchronisation Test

+
    +
  • A comparison is made between your local APEL database, and the data that you have published to the GOC. +
  • Major differences are flagged with FAIL. +
  • Information about APEL APEL Wiki +
  • Contact: apel-admins [at] stfc.ac.uk +
  • lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}
+ + + + + + + + + + + + + + {% for record in records %} + + + + + + + + + + + {% endfor %} + +
All sites
Site NameMonthRecord StartRecord EndRecord Count
In Your Database		Record Count
What You Published		Synchronisation
Status
{{ record.SiteName }}{{ record.YearMonth }} {{ record.RecordStart }}{{ record.RecordEnd }}{{ record.RecordCountPublished }}{{ record.RecordCountInDb }}{{ record.SyncStatus }}
\ No newline at end of file diff --git a/monitoring/publishing/templates/gridsync_singlesite.html b/monitoring/publishing/templates/gridsync_singlesite.html new file mode 100644 index 0000000..e079a0b --- /dev/null +++ b/monitoring/publishing/templates/gridsync_singlesite.html @@ -0,0 +1,41 @@ + + + + + APEL Publication Summary + + + + +

APEL Synchronisation Test

+
    +
  • A comparison is made between your local APEL database, and the data that you have published to the GOC. +
  • Major differences are flagged with FAIL. +
  • Information about APEL APEL Wiki +
  • Contact: apel-admins [at] stfc.ac.uk +
  • lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}
+ + + + + + + + + + + + + {% for record in records %} + + + + + + + + + + {% endfor %} + +
{{ records.0.SiteName }}
MonthRecord StartRecord EndRecord Count
In Your Database		Record Count
What You Published		Synchronisation
Status
{{ record.YearMonth }} {{ record.RecordStart }}{{ record.RecordEnd }}{{ record.RecordCountPublished }}{{ record.RecordCountInDb }}{{ record.SyncStatus }}
\ No newline at end of file From 3a231a861d51db7152681463bceb4af53bb21979 Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 10:50:05 +0000 Subject: [PATCH 051/183] Update field names to ensure consistency with modified views. --- monitoring/publishing/templates/cloudsites.html | 6 +++--- monitoring/publishing/templates/gridsites.html | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/monitoring/publishing/templates/cloudsites.html b/monitoring/publishing/templates/cloudsites.html index e9c2bb7..5fa8680 100644 --- a/monitoring/publishing/templates/cloudsites.html +++ b/monitoring/publishing/templates/cloudsites.html @@ -16,9 +16,9 @@

Sites publishing cloud accounting records from 2018-06-19 onwards

{% for site in sites %} - {{ site.name }} - {{ site.vms }} - {{ site.script }} + {{ site.SiteName }} + {{ site.Vms }} + {{ site.Script }} {{ site.updated|date:"Y-m-d H:i:s" }} {% endfor %} diff --git a/monitoring/publishing/templates/gridsites.html b/monitoring/publishing/templates/gridsites.html index ad5d5a5..0ec21b0 100644 --- a/monitoring/publishing/templates/gridsites.html +++ b/monitoring/publishing/templates/gridsites.html @@ -17,7 +17,7 @@

APEL Publication Test

  • lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}
    - + @@ -25,7 +25,7 @@

    APEL Publication Test

    - + From acecbd3d612c44b3f0d7cab7cf7fa56da8c2e9d0 Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 11:11:28 +0000 Subject: [PATCH 052/183] Add urls for gridsync pages. --- monitoring/publishing/urls.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py index 9691e8d..942c798 100644 --- a/monitoring/publishing/urls.py +++ b/monitoring/publishing/urls.py @@ -3,12 +3,16 @@ from rest_framework import routers from monitoring.publishing import views +from django.urls import re_path router = routers.SimpleRouter() router.register(r'cloud', views.CloudSiteViewSet) router.register(r'grid', views.GridSiteViewSet) +router.register(r'gridsync', views.GridSiteSyncViewSet) - +# Necessary to retrieve one site only urlpatterns = [ - url(r'^', include(router.urls)), + re_path(r'^gridsync/(?P[a-zA-Z0-9-]+)/$', views.GridSiteSyncViewSet.as_view({'get': 'retrieve'}), name='gridsync_singlesite'), ] + +urlpatterns += router.urls From fc13eb71c4cee229ef9d6af2742c3e32c4a14544 Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 15:51:46 +0000 Subject: [PATCH 053/183] Add model for the table with submithosts. --- monitoring/publishing/models.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py index f0e1eb7..1978624 100644 --- a/monitoring/publishing/models.py +++ b/monitoring/publishing/models.py @@ -78,4 +78,20 @@ def __str__(self): self.CloudType, self.UpdateTime, self.VMs) - \ No newline at end of file + + +class GridSiteSyncSubmitH(models.Model): + fetched = models.DateTimeField(auto_now=True) + SiteName = models.CharField(max_length=255) + YearMonth = models.CharField(max_length=255) + Year = models.IntegerField() + Month = models.IntegerField() + RecordStart = models.DateTimeField() + RecordEnd = models.DateTimeField() + RecordCountPublished = models.IntegerField() + RecordCountInDb = models.IntegerField() + SubmitHost = models.CharField(max_length=255) + + class Meta: + ordering = ('SiteName', '-Year', '-Month') + unique_together = ('SiteName', 'YearMonth', 'SubmitHost') \ No newline at end of file From 2a2caea72442b0903ff645e132f0a62f88afc312 Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 15:52:27 +0000 Subject: [PATCH 054/183] Add serialiser for the table with submithosts. --- monitoring/publishing/serializers.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 3da01dc..18bf877 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from monitoring.publishing.models import CloudSite, GridSite, GridSiteSync +from monitoring.publishing.models import CloudSite, GridSite, GridSiteSync, GridSiteSyncSubmitH class GridSiteSerializer(serializers.HyperlinkedModelSerializer): @@ -39,3 +39,15 @@ class CloudSiteSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = CloudSite fields = ('url', 'SiteName', 'Vms', 'Script', 'updated') + + +class GridSiteSyncSubmitHSerializer(serializers.HyperlinkedModelSerializer): + # Override default format with None so that Python datetime is used as + # ouput format. Encoding will be determined by the renderer and can be + # formatted by a template filter. + + class Meta: + model = GridSiteSyncSubmitH + fields = ('url', 'SiteName', 'YearMonth', 'RecordStart', 'RecordEnd', 'RecordCountPublished', 'RecordCountInDb', 'SubmitHost') + + lookup_fields = ('SiteName', 'YearMonth') \ No newline at end of file From 247509365772ef3f812094df811506540aa6cc1b Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 15:54:26 +0000 Subject: [PATCH 055/183] Add view for the sync table showing submithosts. --- monitoring/publishing/views.py | 115 ++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index f3f456a..248df2c 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -9,8 +9,8 @@ from rest_framework import viewsets, generics from rest_framework.renderers import TemplateHTMLRenderer -from monitoring.publishing.models import GridSite, VSuperSummaries, CloudSite, VAnonCloudRecord, GridSiteSync, VSyncRecords -from monitoring.publishing.serializers import GridSiteSerializer, CloudSiteSerializer, GridSiteSyncSerializer +from monitoring.publishing.models import GridSite, VSuperSummaries, CloudSite, VAnonCloudRecord, GridSiteSync, VSyncRecords, GridSiteSyncSubmitH +from monitoring.publishing.serializers import GridSiteSerializer, CloudSiteSerializer, GridSiteSyncSerializer, GridSiteSyncSubmitHSerializer class GridSiteViewSet(viewsets.ReadOnlyModelViewSet): @@ -257,6 +257,117 @@ def retrieve(self, request, SiteName=None): return response +# Needed for passing two parameters to a viewset (GridSiteSyncSubmitHViewSet) +class MultipleFieldLookupMixin: + """ + Apply this mixin to any view or viewset to get multiple field filtering + based on a `lookup_fields` attribute, instead of the default single field filtering. + """ + def get_object(self): + queryset = self.get_queryset() + queryset = self.filter_queryset(queryset) + filter = {} + for field in self.lookup_fields: + if self.kwargs.get(field): + filter[field] = self.kwargs[field] + obj = get_object_or_404(queryset, **filter) + self.check_object_permissions(self.request, obj) + return obj + + +class GridSiteSyncSubmitHViewSet(MultipleFieldLookupMixin, viewsets.ReadOnlyModelViewSet): + queryset = GridSiteSyncSubmitH.objects.all() + serializer_class = GridSiteSyncSubmitHSerializer + template_name = 'gridsync_submithost.html' + + def list(self, request): + response = super(GridSiteSyncSubmitHViewSet, self).list(request) + response.data = {'submisthosts': response.data, 'last_fetched': last_fetched} + return response + + def retrieve(self, request, SiteName=None, YearMonth=None): + + lookup_fields = ('SiteName', 'YearMonth') + last_fetched = GridSiteSyncSubmitH.objects.aggregate(Max('fetched'))['fetched__max'] + Year, Month = YearMonth.replace('-', ' ').split(' ') + sitename_in_table = None + yearmonth_in_table = None + + # This is to ensure the data is updated when changing (or clicking on another) month + if GridSiteSyncSubmitH.objects.count() > 0: + row_1 = GridSiteSyncSubmitH.objects.filter()[:1].get() + sitename_in_table = row_1.SiteName + yearmonth_in_table = row_1.YearMonth + + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or (sitename_in_table!=SiteName) or (yearmonth_in_table!=YearMonth) : + print('Out of date') + + fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, SubmitHost AS SubmitHostSumm, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) + fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb, SubmitHost AS SubmitHostSync FROM VSyncRecords WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) + + Summaries_dict = {"Site":[], "Month":[], "Year":[], "SubmitHostSumm":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} + SyncRecords_dict = {"Site":[], "Month":[], "Year":[],"SubmitHostSync":[], "RecordCountInDb":[]} + + for row in fetchset_Summaries: + Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] + Summaries_dict["Month"] = Summaries_dict.get("Month") + [row.Month] + Summaries_dict["Year"] = Summaries_dict.get("Year") + [row.Year] + Summaries_dict["SubmitHostSumm"] = Summaries_dict.get("SubmitHostSumm") + [row.SubmitHostSumm] + Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] + Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] + Summaries_dict["RecordEnd"] = Summaries_dict.get("RecordEnd") + [row.RecordEnd] + + for row in fetchset_SyncRecords: + SyncRecords_dict["Site"] = SyncRecords_dict.get("Site") + [row.Site] + SyncRecords_dict["Month"] = SyncRecords_dict.get("Month") + [row.Month] + SyncRecords_dict["Year"] = SyncRecords_dict.get("Year") + [row.Year] + SyncRecords_dict["SubmitHostSync"] = SyncRecords_dict.get("SubmitHostSync") + [row.SubmitHostSync] + SyncRecords_dict["RecordCountInDb"] = SyncRecords_dict.get("RecordCountInDb") + [row.RecordCountInDb] + + df_Summaries = pd.DataFrame.from_dict(Summaries_dict) + df_SyncRecords = pd.DataFrame.from_dict(SyncRecords_dict) + df_Summaries.dropna(inplace=True) + df_SyncRecords.dropna(inplace=True) + + df_all = df_Summaries.merge(df_SyncRecords, left_on=['Site', 'Month', 'Year', 'SubmitHostSumm'], right_on=['Site', 'Month', 'Year', 'SubmitHostSync'], how='outer') + fetchset = df_all.to_dict('index') + + # This is to list only data for one month + GridSiteSyncSubmitH.objects.all().delete() + + def get_year_month_string(year, month): + year_string = str(year) + month_string = str(month) + if len(month_string)==1: + month_string = '0'+month_string + return year_string+ '-' +month_string + + for f in fetchset.values(): + GridSiteSyncSubmitH.objects.update_or_create( + defaults={ + 'RecordStart': f.get("RecordStart"), + 'RecordEnd': f.get("RecordEnd"), + 'RecordCountPublished': f.get("RecordCountPublished"), + 'RecordCountInDb': f.get("RecordCountInDb"), + }, + SiteName=f.get("Site"), + YearMonth = get_year_month_string(f.get("Year"), f.get("Month")), + Month=f.get("Month"), + Year=f.get("Year"), + SubmitHost = f.get("SubmitHostSumm"), + ) + + else: + print('No need to update') + + response = super(GridSiteSyncSubmitHViewSet, self).list(request) + response.data = {'submisthosts': response.data, 'last_fetched': last_fetched} + return response + + + class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet): queryset = CloudSite.objects.all() serializer_class = CloudSiteSerializer From 6ebb2d153c42a33dca15266d63ab5a283830a67f Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 15:55:50 +0000 Subject: [PATCH 056/183] Add hyperlink to table showing submithosts. --- monitoring/publishing/templates/gridsync_singlesite.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/publishing/templates/gridsync_singlesite.html b/monitoring/publishing/templates/gridsync_singlesite.html index e079a0b..00f7aca 100644 --- a/monitoring/publishing/templates/gridsync_singlesite.html +++ b/monitoring/publishing/templates/gridsync_singlesite.html @@ -28,7 +28,7 @@

    APEL Synchronisation Test

    {% for record in records %}     - + From 671b4abcbc839cf7c217c20a6733f44e2edc6540 Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 15:56:17 +0000 Subject: [PATCH 057/183] Add template for table showing submithosts. --- .../templates/gridsync_submithost.html | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 monitoring/publishing/templates/gridsync_submithost.html diff --git a/monitoring/publishing/templates/gridsync_submithost.html b/monitoring/publishing/templates/gridsync_submithost.html new file mode 100644 index 0000000..a919ea6 --- /dev/null +++ b/monitoring/publishing/templates/gridsync_submithost.html @@ -0,0 +1,43 @@ + + + + + APEL Publication Summary + + + + +

    APEL Synchronisation Test

    +
      +
    • A comparison is made between your local APEL database, and the data that you have published to the GOC. +
    • Major differences are flagged with FAIL. +
    • Information about APEL APEL Wiki +
    • Contact: apel-admins [at] stfc.ac.uk +
    • lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}
    + +
    {{ sites.0.name }}
    {{ sites.0.SiteName }}
    ExecutingSite MeasurementDate Publication
    Status
    {{ sites.0.name }}{{ sites.0.SiteName }} {{ last_fetched|date:"Y-m-d" }} {{ last_fetched|date:"G:i:s" }} {{ stdout }}
    {{ record.YearMonth }} {{ record.YearMonth }} {{ record.RecordStart }} {{ record.RecordEnd }} {{ record.RecordCountPublished }}
    + + + + + + + + + + + {% for host in submisthosts %} + + + + + + + + + + + + {% endfor %} + +
    {{submisthosts.0.SiteName}}, {{submisthosts.0.YearMonth}}
    MonthSubmitHostRecordStartRecordEndRecord Count
    In Your Database    		Record Count
    What You Published
    {{ host.YearMonth }} {{ host.SubmitHost }}{{ host.RecordStart }}{{ host.RecordEnd }}{{ host.RecordCountPublished }}{{ host.RecordCountInDb }}
    \ No newline at end of file From b37dde27df3386f3d455234a701c222453156a15 Mon Sep 17 00:00:00 2001 From: Letizia Date: Wed, 25 Jan 2023 16:04:30 +0000 Subject: [PATCH 058/183] Add url to the page showing submithosts. --- monitoring/publishing/urls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py index 942c798..449ded6 100644 --- a/monitoring/publishing/urls.py +++ b/monitoring/publishing/urls.py @@ -9,10 +9,11 @@ router.register(r'cloud', views.CloudSiteViewSet) router.register(r'grid', views.GridSiteViewSet) router.register(r'gridsync', views.GridSiteSyncViewSet) +router.register(r'gridsync', views.GridSiteSyncSubmitHViewSet) -# Necessary to retrieve one site only urlpatterns = [ re_path(r'^gridsync/(?P[a-zA-Z0-9-]+)/$', views.GridSiteSyncViewSet.as_view({'get': 'retrieve'}), name='gridsync_singlesite'), + re_path(r'^gridsync/(?P[a-zA-Z0-9-]+)/(?P[0-9-]+)/$', views.GridSiteSyncSubmitHViewSet.as_view({'get': 'retrieve'}), name='gridsync_submithost'), ] urlpatterns += router.urls From 7db5eba1cf79654b532ae5bed4df9405be46633b Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 12:13:34 +0000 Subject: [PATCH 059/183] Trim trailing whitespaces. --- monitoring/publishing/models.py | 22 ++--- monitoring/publishing/serializers.py | 2 +- monitoring/publishing/views.py | 124 +++++++++++++-------------- monitoring/settings.py | 2 +- 4 files changed, 75 insertions(+), 75 deletions(-) diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py index 1978624..26bbb69 100644 --- a/monitoring/publishing/models.py +++ b/monitoring/publishing/models.py @@ -15,10 +15,10 @@ class VSuperSummaries(models.Model): LatestPublish = models.DateTimeField() Month = models.IntegerField() Year = models.IntegerField() - RecordStart = models.DateTimeField() - RecordEnd = models.DateTimeField() - RecordCountPublished = models.IntegerField() - + RecordStart = models.DateTimeField() + RecordEnd = models.DateTimeField() + RecordCountPublished = models.IntegerField() + class Meta: managed = False db_table = 'VSuperSummaries' @@ -30,8 +30,8 @@ class GridSiteSync(models.Model): YearMonth = models.CharField(max_length=255) Year = models.IntegerField() Month = models.IntegerField() - RecordStart = models.DateTimeField() - RecordEnd = models.DateTimeField() + RecordStart = models.DateTimeField() + RecordEnd = models.DateTimeField() RecordCountPublished = models.IntegerField() RecordCountInDb = models.IntegerField() SyncStatus = models.CharField(max_length=255) @@ -40,7 +40,7 @@ class Meta: # Descending order of Year and Month to display latest data first ordering = ('SiteName', '-Year', '-Month') unique_together = ('SiteName', 'YearMonth') - + class VSyncRecords(models.Model): Site = models.CharField(max_length=255, primary_key=True) @@ -78,16 +78,16 @@ def __str__(self): self.CloudType, self.UpdateTime, self.VMs) - - + + class GridSiteSyncSubmitH(models.Model): fetched = models.DateTimeField(auto_now=True) SiteName = models.CharField(max_length=255) YearMonth = models.CharField(max_length=255) Year = models.IntegerField() Month = models.IntegerField() - RecordStart = models.DateTimeField() - RecordEnd = models.DateTimeField() + RecordStart = models.DateTimeField() + RecordEnd = models.DateTimeField() RecordCountPublished = models.IntegerField() RecordCountInDb = models.IntegerField() SubmitHost = models.CharField(max_length=255) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 18bf877..f01fee0 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -49,5 +49,5 @@ class GridSiteSyncSubmitHSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = GridSiteSyncSubmitH fields = ('url', 'SiteName', 'YearMonth', 'RecordStart', 'RecordEnd', 'RecordCountPublished', 'RecordCountInDb', 'SubmitHost') - + lookup_fields = ('SiteName', 'YearMonth') \ No newline at end of file diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 248df2c..7abb7a3 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -31,7 +31,7 @@ def list(self, request): final_response = [] response = super(GridSiteViewSet, self).list(request) - + for single_dict in response.data: date = single_dict.get('updated').replace(tzinfo=None) @@ -46,13 +46,13 @@ def list(self, request): single_dict['returncode'] = 3 single_dict['stdout'] = "UNKNOWN" final_response.append(single_dict) - + if type(request.accepted_renderer) is TemplateHTMLRenderer: response.data = {'sites': final_response, 'last_fetched': last_fetched} return response - + def retrieve(self, request, pk=None): last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max'] # If there's no data then last_fetched is None. @@ -92,14 +92,14 @@ def retrieve(self, request, pk=None): class GridSiteSyncViewSet(viewsets.ReadOnlyModelViewSet): queryset = GridSiteSync.objects.all() serializer_class = GridSiteSyncSerializer - lookup_field = 'SiteName' + lookup_field = 'SiteName' - # When a single site is showed (retrieve function used), the template + # When a single site is showed (retrieve function used), the template # is different than the one used when showing a list of sites def get_template_names(self): - if self.action == 'list': + if self.action == 'list': return ['gridsync.html'] - elif self.action == 'retrieve': + elif self.action == 'retrieve': return ['gridsync_singlesite.html'] # Combine Year and Month into one string (display purposes) @@ -110,10 +110,10 @@ def get_year_month_string(self, year, month): month_string = '0'+month_string return year_string+ '-' +month_string - def list(self, request): + def list(self, request): last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] n_sites = GridSiteSync.objects.values('SiteName').distinct().count() - + if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or n_sites==1: @@ -121,25 +121,25 @@ def list(self, request): # The condition on EarliestEndTime and LatestEndTime is necessary to avoid error by pytz because of dates like '00-00-00' fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE EarliestEndTime>'1900-01-01' AND LatestEndTime>'1900-01-01' GROUP BY Site, Year, Month;") - fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords GROUP BY Site, Year, Month") + fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords GROUP BY Site, Year, Month") # Create empty dicts that will become dfs to be combined - Summaries_dict = {"Site":[], "Month":[], "Year":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} - SyncRecords_dict = {"Site":[], "Month":[], "Year":[],"RecordCountInDb":[]} - + Summaries_dict = {"Site":[], "Month":[], "Year":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} + SyncRecords_dict = {"Site":[], "Month":[], "Year":[],"RecordCountInDb":[]} + # Fill the dicts with the fetched data for row in fetchset_Summaries: Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] Summaries_dict["Month"] = Summaries_dict.get("Month") + [row.Month] Summaries_dict["Year"] = Summaries_dict.get("Year") + [row.Year] - Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] - Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] + Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] + Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] Summaries_dict["RecordEnd"] = Summaries_dict.get("RecordEnd") + [row.RecordEnd] for row in fetchset_SyncRecords: - SyncRecords_dict["Site"] = SyncRecords_dict.get("Site") + [row.Site] - SyncRecords_dict["Month"] = SyncRecords_dict.get("Month") + [row.Month] - SyncRecords_dict["Year"] = SyncRecords_dict.get("Year") + [row.Year] + SyncRecords_dict["Site"] = SyncRecords_dict.get("Site") + [row.Site] + SyncRecords_dict["Month"] = SyncRecords_dict.get("Month") + [row.Month] + SyncRecords_dict["Year"] = SyncRecords_dict.get("Year") + [row.Year] SyncRecords_dict["RecordCountInDb"] = SyncRecords_dict.get("RecordCountInDb") + [row.RecordCountInDb] # Merge data from VSuperSummaries and VSyncRecords into one df @@ -163,12 +163,12 @@ def list(self, request): # Combined primary keys outside the default dict GridSiteSync.objects.update_or_create( defaults={ - 'RecordStart': f.get("RecordStart"), - 'RecordEnd': f.get("RecordEnd"), + 'RecordStart': f.get("RecordStart"), + 'RecordEnd': f.get("RecordEnd"), 'RecordCountPublished': f.get("RecordCountPublished"), - 'RecordCountInDb': f.get("RecordCountInDb"), + 'RecordCountInDb': f.get("RecordCountInDb"), 'SyncStatus': f.get("SyncStatus"), - }, + }, YearMonth = self.get_year_month_string(f.get("Year"), f.get("Month")), SiteName=f.get("Site"), Month=f.get("Month"), @@ -183,10 +183,10 @@ def list(self, request): return response - + def retrieve(self, request, SiteName=None): lookup_field = 'SiteName' - last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] + last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] row_1 = GridSiteSync.objects.filter()[:1].get() n_sites = GridSiteSync.objects.values('SiteName').distinct().count() @@ -197,23 +197,23 @@ def retrieve(self, request, SiteName=None): # The condition on EarliestEndTime and LatestEndTime is necessary to avoid error by pytz because of dates like '00-00-00' fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND EarliestEndTime>'1900-01-01' AND LatestEndTime>'1900-01-01'GROUP BY Site, Month, Year;".format(SiteName)) - fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords WHERE Site='{}' GROUP BY Site, Month, Year;".format(SiteName)) + fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords WHERE Site='{}' GROUP BY Site, Month, Year;".format(SiteName)) - Summaries_dict = {"Site":[], "Month":[], "Year":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} - SyncRecords_dict = {"Site":[], "Month":[], "Year":[], "RecordCountInDb":[]} + Summaries_dict = {"Site":[], "Month":[], "Year":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} + SyncRecords_dict = {"Site":[], "Month":[], "Year":[], "RecordCountInDb":[]} for row in fetchset_Summaries: Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] Summaries_dict["Month"] = Summaries_dict.get("Month") + [row.Month] Summaries_dict["Year"] = Summaries_dict.get("Year") + [row.Year] - Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] + Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] Summaries_dict["RecordEnd"] = Summaries_dict.get("RecordEnd") + [row.RecordEnd] for row in fetchset_SyncRecords: - SyncRecords_dict["Site"] = SyncRecords_dict.get("Site") + [row.Site] - SyncRecords_dict["Month"] = SyncRecords_dict.get("Month") + [row.Month] - SyncRecords_dict["Year"] = SyncRecords_dict.get("Year") + [row.Year] + SyncRecords_dict["Site"] = SyncRecords_dict.get("Site") + [row.Site] + SyncRecords_dict["Month"] = SyncRecords_dict.get("Month") + [row.Month] + SyncRecords_dict["Year"] = SyncRecords_dict.get("Year") + [row.Year] SyncRecords_dict["RecordCountInDb"] = SyncRecords_dict.get("RecordCountInDb") + [row.RecordCountInDb] df_Summaries = pd.DataFrame.from_dict(Summaries_dict) @@ -237,12 +237,12 @@ def retrieve(self, request, SiteName=None): GridSiteSync.objects.update_or_create( defaults={ - 'RecordStart': f.get("RecordStart"), - 'RecordEnd': f.get("RecordEnd"), + 'RecordStart': f.get("RecordStart"), + 'RecordEnd': f.get("RecordEnd"), 'RecordCountPublished': f.get("RecordCountPublished"), - 'RecordCountInDb': f.get("RecordCountInDb"), + 'RecordCountInDb': f.get("RecordCountInDb"), 'SyncStatus': f.get("SyncStatus"), - }, + }, YearMonth = self.get_year_month_string(f.get("Year"), f.get("Month")), SiteName=f.get("Site"), Month=f.get("Month"), @@ -255,7 +255,7 @@ def retrieve(self, request, SiteName=None): response = super(GridSiteSyncViewSet, self).list(request) response.data = {'records': response.data, 'last_fetched': last_fetched} return response - + # Needed for passing two parameters to a viewset (GridSiteSyncSubmitHViewSet) class MultipleFieldLookupMixin: @@ -264,13 +264,13 @@ class MultipleFieldLookupMixin: based on a `lookup_fields` attribute, instead of the default single field filtering. """ def get_object(self): - queryset = self.get_queryset() - queryset = self.filter_queryset(queryset) + queryset = self.get_queryset() + queryset = self.filter_queryset(queryset) filter = {} for field in self.lookup_fields: if self.kwargs.get(field): filter[field] = self.kwargs[field] - obj = get_object_or_404(queryset, **filter) + obj = get_object_or_404(queryset, **filter) self.check_object_permissions(self.request, obj) return obj @@ -279,13 +279,13 @@ class GridSiteSyncSubmitHViewSet(MultipleFieldLookupMixin, viewsets.ReadOnlyMode queryset = GridSiteSyncSubmitH.objects.all() serializer_class = GridSiteSyncSubmitHSerializer template_name = 'gridsync_submithost.html' - - def list(self, request): + + def list(self, request): response = super(GridSiteSyncSubmitHViewSet, self).list(request) response.data = {'submisthosts': response.data, 'last_fetched': last_fetched} return response - def retrieve(self, request, SiteName=None, YearMonth=None): + def retrieve(self, request, SiteName=None, YearMonth=None): lookup_fields = ('SiteName', 'YearMonth') last_fetched = GridSiteSyncSubmitH.objects.aggregate(Max('fetched'))['fetched__max'] @@ -293,10 +293,10 @@ def retrieve(self, request, SiteName=None, YearMonth=None): sitename_in_table = None yearmonth_in_table = None - # This is to ensure the data is updated when changing (or clicking on another) month + # This is to ensure the data is updated when changing (or clicking on another) month if GridSiteSyncSubmitH.objects.count() > 0: row_1 = GridSiteSyncSubmitH.objects.filter()[:1].get() - sitename_in_table = row_1.SiteName + sitename_in_table = row_1.SiteName yearmonth_in_table = row_1.YearMonth if last_fetched is not None: @@ -304,26 +304,26 @@ def retrieve(self, request, SiteName=None, YearMonth=None): if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or (sitename_in_table!=SiteName) or (yearmonth_in_table!=YearMonth) : print('Out of date') - fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, SubmitHost AS SubmitHostSumm, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) - fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb, SubmitHost AS SubmitHostSync FROM VSyncRecords WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) + fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, SubmitHost AS SubmitHostSumm, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) + fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb, SubmitHost AS SubmitHostSync FROM VSyncRecords WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) - Summaries_dict = {"Site":[], "Month":[], "Year":[], "SubmitHostSumm":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} - SyncRecords_dict = {"Site":[], "Month":[], "Year":[],"SubmitHostSync":[], "RecordCountInDb":[]} + Summaries_dict = {"Site":[], "Month":[], "Year":[], "SubmitHostSumm":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} + SyncRecords_dict = {"Site":[], "Month":[], "Year":[],"SubmitHostSync":[], "RecordCountInDb":[]} for row in fetchset_Summaries: Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] Summaries_dict["Month"] = Summaries_dict.get("Month") + [row.Month] Summaries_dict["Year"] = Summaries_dict.get("Year") + [row.Year] Summaries_dict["SubmitHostSumm"] = Summaries_dict.get("SubmitHostSumm") + [row.SubmitHostSumm] - Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] - Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] + Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] + Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] Summaries_dict["RecordEnd"] = Summaries_dict.get("RecordEnd") + [row.RecordEnd] for row in fetchset_SyncRecords: - SyncRecords_dict["Site"] = SyncRecords_dict.get("Site") + [row.Site] - SyncRecords_dict["Month"] = SyncRecords_dict.get("Month") + [row.Month] + SyncRecords_dict["Site"] = SyncRecords_dict.get("Site") + [row.Site] + SyncRecords_dict["Month"] = SyncRecords_dict.get("Month") + [row.Month] SyncRecords_dict["Year"] = SyncRecords_dict.get("Year") + [row.Year] - SyncRecords_dict["SubmitHostSync"] = SyncRecords_dict.get("SubmitHostSync") + [row.SubmitHostSync] + SyncRecords_dict["SubmitHostSync"] = SyncRecords_dict.get("SubmitHostSync") + [row.SubmitHostSync] SyncRecords_dict["RecordCountInDb"] = SyncRecords_dict.get("RecordCountInDb") + [row.RecordCountInDb] df_Summaries = pd.DataFrame.from_dict(Summaries_dict) @@ -333,10 +333,10 @@ def retrieve(self, request, SiteName=None, YearMonth=None): df_all = df_Summaries.merge(df_SyncRecords, left_on=['Site', 'Month', 'Year', 'SubmitHostSumm'], right_on=['Site', 'Month', 'Year', 'SubmitHostSync'], how='outer') fetchset = df_all.to_dict('index') - - # This is to list only data for one month + + # This is to list only data for one month GridSiteSyncSubmitH.objects.all().delete() - + def get_year_month_string(year, month): year_string = str(year) month_string = str(month) @@ -347,13 +347,13 @@ def get_year_month_string(year, month): for f in fetchset.values(): GridSiteSyncSubmitH.objects.update_or_create( defaults={ - 'RecordStart': f.get("RecordStart"), - 'RecordEnd': f.get("RecordEnd"), + 'RecordStart': f.get("RecordStart"), + 'RecordEnd': f.get("RecordEnd"), 'RecordCountPublished': f.get("RecordCountPublished"), - 'RecordCountInDb': f.get("RecordCountInDb"), - }, + 'RecordCountInDb': f.get("RecordCountInDb"), + }, SiteName=f.get("Site"), - YearMonth = get_year_month_string(f.get("Year"), f.get("Month")), + YearMonth = get_year_month_string(f.get("Year"), f.get("Month")), Month=f.get("Month"), Year=f.get("Year"), SubmitHost = f.get("SubmitHostSumm"), @@ -365,7 +365,7 @@ def get_year_month_string(year, month): response = super(GridSiteSyncSubmitHViewSet, self).list(request) response.data = {'submisthosts': response.data, 'last_fetched': last_fetched} return response - + class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet): diff --git a/monitoring/settings.py b/monitoring/settings.py index e1da566..89849fb 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -143,4 +143,4 @@ # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' -STATIC_ROOT = os.path.join(BASE_DIR, "static/") \ No newline at end of file +STATIC_ROOT = os.path.join(BASE_DIR, "static/") \ No newline at end of file From bc1a8ac86d7191d30602b07c714aa97ad33d2564 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 12:20:13 +0000 Subject: [PATCH 060/183] Add newline at end of file. --- monitoring/__init__.py | 2 +- monitoring/publishing/models.py | 2 +- monitoring/publishing/serializers.py | 2 +- monitoring/publishing/views.py | 2 +- monitoring/settings.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/monitoring/__init__.py b/monitoring/__init__.py index c45523b..063cd2c 100644 --- a/monitoring/__init__.py +++ b/monitoring/__init__.py @@ -1,2 +1,2 @@ import pymysql -pymysql.install_as_MySQLdb() \ No newline at end of file +pymysql.install_as_MySQLdb() diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py index 26bbb69..195872d 100644 --- a/monitoring/publishing/models.py +++ b/monitoring/publishing/models.py @@ -94,4 +94,4 @@ class GridSiteSyncSubmitH(models.Model): class Meta: ordering = ('SiteName', '-Year', '-Month') - unique_together = ('SiteName', 'YearMonth', 'SubmitHost') \ No newline at end of file + unique_together = ('SiteName', 'YearMonth', 'SubmitHost') diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index f01fee0..20ed09f 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -50,4 +50,4 @@ class Meta: model = GridSiteSyncSubmitH fields = ('url', 'SiteName', 'YearMonth', 'RecordStart', 'RecordEnd', 'RecordCountPublished', 'RecordCountInDb', 'SubmitHost') - lookup_fields = ('SiteName', 'YearMonth') \ No newline at end of file + lookup_fields = ('SiteName', 'YearMonth') diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 7abb7a3..0fe7ac0 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -411,4 +411,4 @@ def retrieve(self, request, pk=None): response.data['returncode'] = 3 response.data['stdout'] = "UNKNOWN" - return response \ No newline at end of file + return response diff --git a/monitoring/settings.py b/monitoring/settings.py index 89849fb..a476471 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -143,4 +143,4 @@ # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' -STATIC_ROOT = os.path.join(BASE_DIR, "static/") \ No newline at end of file +STATIC_ROOT = os.path.join(BASE_DIR, "static/") From c611943672ba4888a6b8380b52865bd061589fbb Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 12:30:16 +0000 Subject: [PATCH 061/183] Remove whitespace(s) where not necessary or too many. --- monitoring/publishing/views.py | 26 +++++++++++++------------- monitoring/settings.py | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 0fe7ac0..85f5b51 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -124,16 +124,16 @@ def list(self, request): fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords GROUP BY Site, Year, Month") # Create empty dicts that will become dfs to be combined - Summaries_dict = {"Site":[], "Month":[], "Year":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} - SyncRecords_dict = {"Site":[], "Month":[], "Year":[],"RecordCountInDb":[]} + Summaries_dict = {"Site":[], "Month":[], "Year":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} + SyncRecords_dict = {"Site":[], "Month":[], "Year":[],"RecordCountInDb":[]} # Fill the dicts with the fetched data for row in fetchset_Summaries: Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] Summaries_dict["Month"] = Summaries_dict.get("Month") + [row.Month] Summaries_dict["Year"] = Summaries_dict.get("Year") + [row.Year] - Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] - Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] + Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] + Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] Summaries_dict["RecordEnd"] = Summaries_dict.get("RecordEnd") + [row.RecordEnd] for row in fetchset_SyncRecords: @@ -199,15 +199,15 @@ def retrieve(self, request, SiteName=None): fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND EarliestEndTime>'1900-01-01' AND LatestEndTime>'1900-01-01'GROUP BY Site, Month, Year;".format(SiteName)) fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords WHERE Site='{}' GROUP BY Site, Month, Year;".format(SiteName)) - Summaries_dict = {"Site":[], "Month":[], "Year":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} - SyncRecords_dict = {"Site":[], "Month":[], "Year":[], "RecordCountInDb":[]} + Summaries_dict = {"Site":[], "Month":[], "Year":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} + SyncRecords_dict = {"Site":[], "Month":[], "Year":[], "RecordCountInDb":[]} for row in fetchset_Summaries: Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] Summaries_dict["Month"] = Summaries_dict.get("Month") + [row.Month] Summaries_dict["Year"] = Summaries_dict.get("Year") + [row.Year] - Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] - Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] + Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] + Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] Summaries_dict["RecordEnd"] = Summaries_dict.get("RecordEnd") + [row.RecordEnd] for row in fetchset_SyncRecords: @@ -301,22 +301,22 @@ def retrieve(self, request, SiteName=None, YearMonth=None): if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or (sitename_in_table!=SiteName) or (yearmonth_in_table!=YearMonth) : + if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or (sitename_in_table!=SiteName) or (yearmonth_in_table!=YearMonth): print('Out of date') fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, SubmitHost AS SubmitHostSumm, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb, SubmitHost AS SubmitHostSync FROM VSyncRecords WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) - Summaries_dict = {"Site":[], "Month":[], "Year":[], "SubmitHostSumm":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} - SyncRecords_dict = {"Site":[], "Month":[], "Year":[],"SubmitHostSync":[], "RecordCountInDb":[]} + Summaries_dict = {"Site":[], "Month":[], "Year":[], "SubmitHostSumm":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} + SyncRecords_dict = {"Site":[], "Month":[], "Year":[],"SubmitHostSync":[], "RecordCountInDb":[]} for row in fetchset_Summaries: Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] Summaries_dict["Month"] = Summaries_dict.get("Month") + [row.Month] Summaries_dict["Year"] = Summaries_dict.get("Year") + [row.Year] Summaries_dict["SubmitHostSumm"] = Summaries_dict.get("SubmitHostSumm") + [row.SubmitHostSumm] - Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] - Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] + Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] + Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] Summaries_dict["RecordEnd"] = Summaries_dict.get("RecordEnd") + [row.RecordEnd] for row in fetchset_SyncRecords: diff --git a/monitoring/settings.py b/monitoring/settings.py index a476471..5d3d44f 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -25,7 +25,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [ '127.0.0.1', 'localhost'] +ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] # Application definition From 2dbb9cfaa3b21a8f5269bbceafb01d5a76875288 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 12:38:30 +0000 Subject: [PATCH 062/183] Add whitespace(s) around operators where necessary. --- monitoring/publishing/views.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 85f5b51..be8a1cc 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -106,9 +106,9 @@ def get_template_names(self): def get_year_month_string(self, year, month): year_string = str(year) month_string = str(month) - if len(month_string)==1: - month_string = '0'+month_string - return year_string+ '-' +month_string + if len(month_string) == 1: + month_string = '0' + month_string + return year_string + '-' + month_string def list(self, request): last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] @@ -116,7 +116,7 @@ def list(self, request): if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or n_sites==1: + if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or n_sites == 1: print('Out of date') # The condition on EarliestEndTime and LatestEndTime is necessary to avoid error by pytz because of dates like '00-00-00' @@ -156,9 +156,9 @@ def list(self, request): rel_diff1 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountInDb")) rel_diff2 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountPublished")) if rel_diff1 < 0.01 or rel_diff2 < 0.01: - f['SyncStatus']='OK' + f['SyncStatus'] = 'OK' else: - f['SyncStatus']='Error' + f['SyncStatus'] = 'Error' # Combined primary keys outside the default dict GridSiteSync.objects.update_or_create( @@ -231,9 +231,9 @@ def retrieve(self, request, SiteName=None): rel_diff1 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountInDb")) rel_diff2 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountPublished")) if rel_diff1 <= 0.01 or rel_diff2 <= 0.01: - f['SyncStatus']='OK' + f['SyncStatus'] = 'OK' else: - f['SyncStatus']='Error' + f['SyncStatus'] = 'Error' GridSiteSync.objects.update_or_create( defaults={ @@ -301,7 +301,7 @@ def retrieve(self, request, SiteName=None, YearMonth=None): if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or (sitename_in_table!=SiteName) or (yearmonth_in_table!=YearMonth): + if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or (sitename_in_table != SiteName) or (yearmonth_in_table != YearMonth): print('Out of date') fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, SubmitHost AS SubmitHostSumm, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) @@ -340,9 +340,9 @@ def retrieve(self, request, SiteName=None, YearMonth=None): def get_year_month_string(year, month): year_string = str(year) month_string = str(month) - if len(month_string)==1: - month_string = '0'+month_string - return year_string+ '-' +month_string + if len(month_string) == 1: + month_string = '0' + month_string + return year_string + '-' + month_string for f in fetchset.values(): GridSiteSyncSubmitH.objects.update_or_create( From d55d6ab339929cf2982e355970d8a0e1f0d7b593 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 12:47:26 +0000 Subject: [PATCH 063/183] Remove spaces around keyword / parameter equals. --- monitoring/publishing/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index be8a1cc..b4b6856 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -169,7 +169,7 @@ def list(self, request): 'RecordCountInDb': f.get("RecordCountInDb"), 'SyncStatus': f.get("SyncStatus"), }, - YearMonth = self.get_year_month_string(f.get("Year"), f.get("Month")), + YearMonth=self.get_year_month_string(f.get("Year"), f.get("Month")), SiteName=f.get("Site"), Month=f.get("Month"), Year=f.get("Year"), @@ -243,7 +243,7 @@ def retrieve(self, request, SiteName=None): 'RecordCountInDb': f.get("RecordCountInDb"), 'SyncStatus': f.get("SyncStatus"), }, - YearMonth = self.get_year_month_string(f.get("Year"), f.get("Month")), + YearMonth=self.get_year_month_string(f.get("Year"), f.get("Month")), SiteName=f.get("Site"), Month=f.get("Month"), Year=f.get("Year"), @@ -353,10 +353,10 @@ def get_year_month_string(year, month): 'RecordCountInDb': f.get("RecordCountInDb"), }, SiteName=f.get("Site"), - YearMonth = get_year_month_string(f.get("Year"), f.get("Month")), + YearMonth=get_year_month_string(f.get("Year"), f.get("Month")), Month=f.get("Month"), Year=f.get("Year"), - SubmitHost = f.get("SubmitHostSumm"), + SubmitHost=f.get("SubmitHostSumm"), ) else: From 964d49886d9f2b288feb867d2187a1d49f55de30 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 12:56:23 +0000 Subject: [PATCH 064/183] Add space after punctuation where necessary. --- monitoring/publishing/views.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index b4b6856..3a99421 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -124,8 +124,8 @@ def list(self, request): fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords GROUP BY Site, Year, Month") # Create empty dicts that will become dfs to be combined - Summaries_dict = {"Site":[], "Month":[], "Year":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} - SyncRecords_dict = {"Site":[], "Month":[], "Year":[],"RecordCountInDb":[]} + Summaries_dict = {"Site": [], "Month": [], "Year": [], "RecordCountPublished": [], "RecordStart": [], "RecordEnd": []} + SyncRecords_dict = {"Site": [], "Month": [], "Year": [], "RecordCountInDb": []} # Fill the dicts with the fetched data for row in fetchset_Summaries: @@ -199,8 +199,8 @@ def retrieve(self, request, SiteName=None): fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND EarliestEndTime>'1900-01-01' AND LatestEndTime>'1900-01-01'GROUP BY Site, Month, Year;".format(SiteName)) fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords WHERE Site='{}' GROUP BY Site, Month, Year;".format(SiteName)) - Summaries_dict = {"Site":[], "Month":[], "Year":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} - SyncRecords_dict = {"Site":[], "Month":[], "Year":[], "RecordCountInDb":[]} + Summaries_dict = {"Site": [], "Month": [], "Year": [], "RecordCountPublished": [], "RecordStart": [], "RecordEnd": []} + SyncRecords_dict = {"Site": [], "Month": [], "Year": [], "RecordCountInDb": []} for row in fetchset_Summaries: Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] @@ -307,8 +307,8 @@ def retrieve(self, request, SiteName=None, YearMonth=None): fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, SubmitHost AS SubmitHostSumm, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb, SubmitHost AS SubmitHostSync FROM VSyncRecords WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) - Summaries_dict = {"Site":[], "Month":[], "Year":[], "SubmitHostSumm":[], "RecordCountPublished":[],"RecordStart":[], "RecordEnd":[]} - SyncRecords_dict = {"Site":[], "Month":[], "Year":[],"SubmitHostSync":[], "RecordCountInDb":[]} + Summaries_dict = {"Site": [], "Month": [], "Year": [], "SubmitHostSumm": [], "RecordCountPublished": [], "RecordStart": [], "RecordEnd": []} + SyncRecords_dict = {"Site": [], "Month": [], "Year": [], "SubmitHostSync": [], "RecordCountInDb": []} for row in fetchset_Summaries: Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] From 581cd046bd4997d9d05c6646c4da3bc56c2aa61b Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 13:02:54 +0000 Subject: [PATCH 065/183] Remove unnecessary blank lines. --- monitoring/publishing/views.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 3a99421..8843976 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -52,7 +52,6 @@ def list(self, request): return response - def retrieve(self, request, pk=None): last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max'] # If there's no data then last_fetched is None. @@ -88,7 +87,6 @@ def retrieve(self, request, pk=None): return response - class GridSiteSyncViewSet(viewsets.ReadOnlyModelViewSet): queryset = GridSiteSync.objects.all() serializer_class = GridSiteSyncSerializer @@ -182,8 +180,6 @@ def list(self, request): response.data = {'records': response.data, 'last_fetched': last_fetched} return response - - def retrieve(self, request, SiteName=None): lookup_field = 'SiteName' last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] @@ -367,7 +363,6 @@ def get_year_month_string(year, month): return response - class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet): queryset = CloudSite.objects.all() serializer_class = CloudSiteSerializer From 2d3f9610d9522b0a12ac0acc3a7305614ae08927 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 13:05:43 +0000 Subject: [PATCH 066/183] Move import to the top of file. --- monitoring/availability/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/availability/urls.py b/monitoring/availability/urls.py index edf402e..12d45b8 100644 --- a/monitoring/availability/urls.py +++ b/monitoring/availability/urls.py @@ -1,9 +1,9 @@ from django.conf.urls import url - import sys -sys.path.append('/usr/share/DJANGO_MONITORING_APP') from monitoring.availability import views +sys.path.append('/usr/share/DJANGO_MONITORING_APP') + urlpatterns = [ url(r'^$', views.status), ] From 58af9a8e8d4e6be0723fe0e8982ba146e7be3c14 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 13:08:39 +0000 Subject: [PATCH 067/183] Rearrange/refactor imports to shorten lines. --- monitoring/publishing/serializers.py | 7 ++++++- monitoring/publishing/views.py | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 20ed09f..23b74b0 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -1,6 +1,11 @@ from rest_framework import serializers -from monitoring.publishing.models import CloudSite, GridSite, GridSiteSync, GridSiteSyncSubmitH +from monitoring.publishing.models import ( + CloudSite, + GridSite, + GridSiteSync, + GridSiteSyncSubmitH +) class GridSiteSerializer(serializers.HyperlinkedModelSerializer): diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 8843976..cb88d52 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -9,8 +9,22 @@ from rest_framework import viewsets, generics from rest_framework.renderers import TemplateHTMLRenderer -from monitoring.publishing.models import GridSite, VSuperSummaries, CloudSite, VAnonCloudRecord, GridSiteSync, VSyncRecords, GridSiteSyncSubmitH -from monitoring.publishing.serializers import GridSiteSerializer, CloudSiteSerializer, GridSiteSyncSerializer, GridSiteSyncSubmitHSerializer +from monitoring.publishing.models import ( + GridSite, + VSuperSummaries, + CloudSite, + VAnonCloudRecord, + GridSiteSync, + VSyncRecords, + GridSiteSyncSubmitH +) + +from monitoring.publishing.serializers import ( + GridSiteSerializer, + CloudSiteSerializer, + GridSiteSyncSerializer, + GridSiteSyncSubmitHSerializer +) class GridSiteViewSet(viewsets.ReadOnlyModelViewSet): From 0cc3f30056aa34b2544c59782b83c957bf62c18a Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 13:10:22 +0000 Subject: [PATCH 068/183] Rearrange/refactor serialiser fields to shorten lines. --- monitoring/publishing/serializers.py | 36 ++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 23b74b0..de73fd1 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -16,7 +16,11 @@ class GridSiteSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = GridSite - fields = ('url', 'SiteName', 'updated') + fields = ( + 'url', + 'SiteName', + 'updated' + ) class GridSiteSyncSerializer(serializers.HyperlinkedModelSerializer): @@ -26,7 +30,16 @@ class GridSiteSyncSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = GridSiteSync - fields = ('url', 'SiteName', 'YearMonth', 'RecordStart', 'RecordEnd', 'RecordCountPublished', 'RecordCountInDb', 'SyncStatus') + fields = ( + 'url', + 'SiteName', + 'YearMonth', + 'RecordStart', + 'RecordEnd', + 'RecordCountPublished', + 'RecordCountInDb', + 'SyncStatus' + ) # Sitename substitutes pk lookup_field = 'SiteName' @@ -43,7 +56,13 @@ class CloudSiteSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = CloudSite - fields = ('url', 'SiteName', 'Vms', 'Script', 'updated') + fields = ( + 'url', + 'SiteName', + 'Vms', + 'Script', + 'updated' + ) class GridSiteSyncSubmitHSerializer(serializers.HyperlinkedModelSerializer): @@ -53,6 +72,15 @@ class GridSiteSyncSubmitHSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = GridSiteSyncSubmitH - fields = ('url', 'SiteName', 'YearMonth', 'RecordStart', 'RecordEnd', 'RecordCountPublished', 'RecordCountInDb', 'SubmitHost') + fields = ( + 'url', + 'SiteName', + 'YearMonth', + 'RecordStart', + 'RecordEnd', + 'RecordCountPublished', + 'RecordCountInDb', + 'SubmitHost' + ) lookup_fields = ('SiteName', 'YearMonth') From 0933fb800098bc3e1996f007ec904d0b39954561 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 13:15:36 +0000 Subject: [PATCH 069/183] Rearrange/refactor urlpatterns to shorten lines. --- monitoring/publishing/urls.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py index 449ded6..7f8d4e4 100644 --- a/monitoring/publishing/urls.py +++ b/monitoring/publishing/urls.py @@ -12,8 +12,16 @@ router.register(r'gridsync', views.GridSiteSyncSubmitHViewSet) urlpatterns = [ - re_path(r'^gridsync/(?P[a-zA-Z0-9-]+)/$', views.GridSiteSyncViewSet.as_view({'get': 'retrieve'}), name='gridsync_singlesite'), - re_path(r'^gridsync/(?P[a-zA-Z0-9-]+)/(?P[0-9-]+)/$', views.GridSiteSyncSubmitHViewSet.as_view({'get': 'retrieve'}), name='gridsync_submithost'), + re_path( + r'^gridsync/(?P[a-zA-Z0-9-]+)/$', + views.GridSiteSyncViewSet.as_view({'get': 'retrieve'}), + name='gridsync_singlesite' + ), + re_path( + r'^gridsync/(?P[a-zA-Z0-9-]+)/(?P[0-9-]+)/$', + views.GridSiteSyncSubmitHViewSet.as_view({'get': 'retrieve'}), + name='gridsync_submithost' + ), ] urlpatterns += router.urls From 6982316e8c7b5d8211506c35d3f205ba3e5c0e09 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 13:38:35 +0000 Subject: [PATCH 070/183] Rearrange dicts to shorten lines. --- monitoring/publishing/views.py | 127 +++++++++++++++++++++++++-------- 1 file changed, 96 insertions(+), 31 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index cb88d52..818f61f 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -62,7 +62,10 @@ def list(self, request): final_response.append(single_dict) if type(request.accepted_renderer) is TemplateHTMLRenderer: - response.data = {'sites': final_response, 'last_fetched': last_fetched} + response.data = { + 'sites': final_response, + 'last_fetched': last_fetched + } return response @@ -85,7 +88,10 @@ def retrieve(self, request, pk=None): # Wrap data in a dict so that it can display in template. if type(request.accepted_renderer) is TemplateHTMLRenderer: # Single result put in list to work with same HTML template. - response.data = {'sites': [response.data], 'last_fetched': last_fetched} + response.data = { + 'sites': [response.data], + 'last_fetched': last_fetched + } diff = datetime.today() - date if diff <= timedelta(days=7): @@ -136,8 +142,21 @@ def list(self, request): fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords GROUP BY Site, Year, Month") # Create empty dicts that will become dfs to be combined - Summaries_dict = {"Site": [], "Month": [], "Year": [], "RecordCountPublished": [], "RecordStart": [], "RecordEnd": []} - SyncRecords_dict = {"Site": [], "Month": [], "Year": [], "RecordCountInDb": []} + Summaries_dict = { + "Site": [], + "Month": [], + "Year": [], + "RecordCountPublished": [], + "RecordStart": [], + "RecordEnd": [] + } + + SyncRecords_dict = { + "Site": [], + "Month": [], + "Year": [], + "RecordCountInDb": [] + } # Fill the dicts with the fetched data for row in fetchset_Summaries: @@ -175,12 +194,12 @@ def list(self, request): # Combined primary keys outside the default dict GridSiteSync.objects.update_or_create( defaults={ - 'RecordStart': f.get("RecordStart"), - 'RecordEnd': f.get("RecordEnd"), - 'RecordCountPublished': f.get("RecordCountPublished"), - 'RecordCountInDb': f.get("RecordCountInDb"), - 'SyncStatus': f.get("SyncStatus"), - }, + 'RecordStart': f.get("RecordStart"), + 'RecordEnd': f.get("RecordEnd"), + 'RecordCountPublished': f.get("RecordCountPublished"), + 'RecordCountInDb': f.get("RecordCountInDb"), + 'SyncStatus': f.get("SyncStatus"), + }, YearMonth=self.get_year_month_string(f.get("Year"), f.get("Month")), SiteName=f.get("Site"), Month=f.get("Month"), @@ -191,7 +210,10 @@ def list(self, request): print('No need to update') response = super(GridSiteSyncViewSet, self).list(request) - response.data = {'records': response.data, 'last_fetched': last_fetched} + response.data = { + 'records': response.data, + 'last_fetched': last_fetched + } return response def retrieve(self, request, SiteName=None): @@ -209,8 +231,21 @@ def retrieve(self, request, SiteName=None): fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND EarliestEndTime>'1900-01-01' AND LatestEndTime>'1900-01-01'GROUP BY Site, Month, Year;".format(SiteName)) fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords WHERE Site='{}' GROUP BY Site, Month, Year;".format(SiteName)) - Summaries_dict = {"Site": [], "Month": [], "Year": [], "RecordCountPublished": [], "RecordStart": [], "RecordEnd": []} - SyncRecords_dict = {"Site": [], "Month": [], "Year": [], "RecordCountInDb": []} + Summaries_dict = { + "Site": [], + "Month": [], + "Year": [], + "RecordCountPublished": [], + "RecordStart": [], + "RecordEnd": [] + } + + SyncRecords_dict = { + "Site": [], + "Month": [], + "Year": [], + "RecordCountInDb": [] + } for row in fetchset_Summaries: Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] @@ -247,12 +282,12 @@ def retrieve(self, request, SiteName=None): GridSiteSync.objects.update_or_create( defaults={ - 'RecordStart': f.get("RecordStart"), - 'RecordEnd': f.get("RecordEnd"), - 'RecordCountPublished': f.get("RecordCountPublished"), - 'RecordCountInDb': f.get("RecordCountInDb"), - 'SyncStatus': f.get("SyncStatus"), - }, + 'RecordStart': f.get("RecordStart"), + 'RecordEnd': f.get("RecordEnd"), + 'RecordCountPublished': f.get("RecordCountPublished"), + 'RecordCountInDb': f.get("RecordCountInDb"), + 'SyncStatus': f.get("SyncStatus"), + }, YearMonth=self.get_year_month_string(f.get("Year"), f.get("Month")), SiteName=f.get("Site"), Month=f.get("Month"), @@ -263,7 +298,10 @@ def retrieve(self, request, SiteName=None): print('No need to update') response = super(GridSiteSyncViewSet, self).list(request) - response.data = {'records': response.data, 'last_fetched': last_fetched} + response.data = { + 'records': response.data, + 'last_fetched': last_fetched + } return response @@ -292,7 +330,10 @@ class GridSiteSyncSubmitHViewSet(MultipleFieldLookupMixin, viewsets.ReadOnlyMode def list(self, request): response = super(GridSiteSyncSubmitHViewSet, self).list(request) - response.data = {'submisthosts': response.data, 'last_fetched': last_fetched} + response.data = { + 'submisthosts': response.data, + 'last_fetched': last_fetched + } return response def retrieve(self, request, SiteName=None, YearMonth=None): @@ -317,8 +358,23 @@ def retrieve(self, request, SiteName=None, YearMonth=None): fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, SubmitHost AS SubmitHostSumm, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb, SubmitHost AS SubmitHostSync FROM VSyncRecords WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) - Summaries_dict = {"Site": [], "Month": [], "Year": [], "SubmitHostSumm": [], "RecordCountPublished": [], "RecordStart": [], "RecordEnd": []} - SyncRecords_dict = {"Site": [], "Month": [], "Year": [], "SubmitHostSync": [], "RecordCountInDb": []} + Summaries_dict = { + "Site": [], + "Month": [], + "Year": [], + "SubmitHostSumm": [], + "RecordCountPublished": [], + "RecordStart": [], + "RecordEnd": [] + } + + SyncRecords_dict = { + "Site": [], + "Month": [], + "Year": [], + "SubmitHostSync": [], + "RecordCountInDb": [] + } for row in fetchset_Summaries: Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] @@ -357,11 +413,11 @@ def get_year_month_string(year, month): for f in fetchset.values(): GridSiteSyncSubmitH.objects.update_or_create( defaults={ - 'RecordStart': f.get("RecordStart"), - 'RecordEnd': f.get("RecordEnd"), - 'RecordCountPublished': f.get("RecordCountPublished"), - 'RecordCountInDb': f.get("RecordCountInDb"), - }, + 'RecordStart': f.get("RecordStart"), + 'RecordEnd': f.get("RecordEnd"), + 'RecordCountPublished': f.get("RecordCountPublished"), + 'RecordCountInDb': f.get("RecordCountInDb"), + }, SiteName=f.get("Site"), YearMonth=get_year_month_string(f.get("Year"), f.get("Month")), Month=f.get("Month"), @@ -373,7 +429,10 @@ def get_year_month_string(year, month): print('No need to update') response = super(GridSiteSyncSubmitHViewSet, self).list(request) - response.data = {'submisthosts': response.data, 'last_fetched': last_fetched} + response.data = { + 'submisthosts': response.data, + 'last_fetched': last_fetched + } return response @@ -397,7 +456,10 @@ def list(self, request): response = super(CloudSiteViewSet, self).list(request) # Wrap data in a dict so that it can display in template. if type(request.accepted_renderer) is TemplateHTMLRenderer: - response.data = {'sites': response.data, 'last_fetched': last_fetched} + response.data = { + 'sites': response.data, + 'last_fetched': last_fetched + } return response def retrieve(self, request, pk=None): @@ -415,7 +477,10 @@ def retrieve(self, request, pk=None): # Wrap data in a dict so that it can display in template. if type(request.accepted_renderer) is TemplateHTMLRenderer: # Single result put in list to work with same HTML template. - response.data = {'sites': [response.data], 'last_fetched': last_fetched} + response.data = { + 'sites': [response.data], + 'last_fetched': last_fetched + } response.data['returncode'] = 3 response.data['stdout'] = "UNKNOWN" From c10e9e92322c51093ce84a3c028ccea60ef72773 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 14:07:56 +0000 Subject: [PATCH 071/183] Refactor parts to create/fill dicts, to reduce duplication and file length. --- monitoring/publishing/views.py | 165 +++++++++++++++------------------ 1 file changed, 76 insertions(+), 89 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 818f61f..37a1608 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -26,6 +26,57 @@ GridSiteSyncSubmitHSerializer ) +summaries_dict_standard = { + "Site": [], + "Month": [], + "Year": [], + "RecordCountPublished": [], + "RecordStart": [], + "RecordEnd": [], + "SubmitHostSumm": [], +} + +syncrecords_dict_standard = { + "Site": [], + "Month": [], + "Year": [], + "RecordCountInDb": [], + "SubmitHostSync": [] +} + + +def fill_summaries_dict(inpDict, row): + inpDict["Site"] = inpDict.get("Site") + [row.Site] + inpDict["Month"] = inpDict.get("Month") + [row.Month] + inpDict["Year"] = inpDict.get("Year") + [row.Year] + inpDict["RecordCountPublished"] = inpDict.get("RecordCountPublished") + [row.RecordCountPublished] + inpDict["RecordStart"] = inpDict.get("RecordStart") + [row.RecordStart] + inpDict["RecordEnd"] = inpDict.get("RecordEnd") + [row.RecordEnd] + if hasattr(row, "SubmitHostSumm"): + inpDict["SubmitHostSumm"] = inpDict.get("SubmitHostSumm") + [row.SubmitHostSumm] + + return inpDict + + +def fill_syncrecords_dict(inpDict, row): + inpDict["Site"] = inpDict.get("Site") + [row.Site] + inpDict["Month"] = inpDict.get("Month") + [row.Month] + inpDict["Year"] = inpDict.get("Year") + [row.Year] + inpDict["RecordCountInDb"] = inpDict.get("RecordCountInDb") + [row.RecordCountInDb] + if hasattr(row, "SubmitHostSync"): + inpDict["SubmitHostSync"] = inpDict.get("SubmitHostSync") + [row.SubmitHostSync] + return inpDict + + +def correct_dict(inpDict): + keys_to_remove = [] + for key, val in inpDict.items(): + if len(val) == 0: + keys_to_remove.append(key) + for key in keys_to_remove: + inpDict.pop(key) + return inpDict + class GridSiteViewSet(viewsets.ReadOnlyModelViewSet): queryset = GridSite.objects.all() @@ -142,40 +193,20 @@ def list(self, request): fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords GROUP BY Site, Year, Month") # Create empty dicts that will become dfs to be combined - Summaries_dict = { - "Site": [], - "Month": [], - "Year": [], - "RecordCountPublished": [], - "RecordStart": [], - "RecordEnd": [] - } - - SyncRecords_dict = { - "Site": [], - "Month": [], - "Year": [], - "RecordCountInDb": [] - } + summaries_dict = summaries_dict_standard.copy() + syncrecords_dict = syncrecords_dict_standard.copy() # Fill the dicts with the fetched data for row in fetchset_Summaries: - Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] - Summaries_dict["Month"] = Summaries_dict.get("Month") + [row.Month] - Summaries_dict["Year"] = Summaries_dict.get("Year") + [row.Year] - Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] - Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] - Summaries_dict["RecordEnd"] = Summaries_dict.get("RecordEnd") + [row.RecordEnd] - + summaries_dict = fill_summaries_dict(summaries_dict, row) + summaries_dict = correct_dict(summaries_dict) for row in fetchset_SyncRecords: - SyncRecords_dict["Site"] = SyncRecords_dict.get("Site") + [row.Site] - SyncRecords_dict["Month"] = SyncRecords_dict.get("Month") + [row.Month] - SyncRecords_dict["Year"] = SyncRecords_dict.get("Year") + [row.Year] - SyncRecords_dict["RecordCountInDb"] = SyncRecords_dict.get("RecordCountInDb") + [row.RecordCountInDb] + syncrecords_dict = fill_syncrecords_dict(syncrecords_dict, row) + syncrecords_dict = correct_dict(syncrecords_dict) # Merge data from VSuperSummaries and VSyncRecords into one df - df_Summaries = pd.DataFrame.from_dict(Summaries_dict) - df_SyncRecords = pd.DataFrame.from_dict(SyncRecords_dict) + df_Summaries = pd.DataFrame.from_dict(summaries_dict) + df_SyncRecords = pd.DataFrame.from_dict(syncrecords_dict) df_all = df_Summaries.merge(df_SyncRecords, left_on=['Site', 'Month', 'Year'], right_on=['Site', 'Month', 'Year'], how='inner') fetchset = df_all.to_dict('index') @@ -231,38 +262,18 @@ def retrieve(self, request, SiteName=None): fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND EarliestEndTime>'1900-01-01' AND LatestEndTime>'1900-01-01'GROUP BY Site, Month, Year;".format(SiteName)) fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords WHERE Site='{}' GROUP BY Site, Month, Year;".format(SiteName)) - Summaries_dict = { - "Site": [], - "Month": [], - "Year": [], - "RecordCountPublished": [], - "RecordStart": [], - "RecordEnd": [] - } - - SyncRecords_dict = { - "Site": [], - "Month": [], - "Year": [], - "RecordCountInDb": [] - } + summaries_dict = summaries_dict_standard.copy() + syncrecords_dict = syncrecords_dict_standard.copy() for row in fetchset_Summaries: - Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] - Summaries_dict["Month"] = Summaries_dict.get("Month") + [row.Month] - Summaries_dict["Year"] = Summaries_dict.get("Year") + [row.Year] - Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] - Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] - Summaries_dict["RecordEnd"] = Summaries_dict.get("RecordEnd") + [row.RecordEnd] - + summaries_dict = fill_summaries_dict(summaries_dict, row) + summaries_dict = correct_dict(summaries_dict) for row in fetchset_SyncRecords: - SyncRecords_dict["Site"] = SyncRecords_dict.get("Site") + [row.Site] - SyncRecords_dict["Month"] = SyncRecords_dict.get("Month") + [row.Month] - SyncRecords_dict["Year"] = SyncRecords_dict.get("Year") + [row.Year] - SyncRecords_dict["RecordCountInDb"] = SyncRecords_dict.get("RecordCountInDb") + [row.RecordCountInDb] + syncrecords_dict = fill_syncrecords_dict(syncrecords_dict, row) + syncrecords_dict = correct_dict(syncrecords_dict) - df_Summaries = pd.DataFrame.from_dict(Summaries_dict) - df_SyncRecords = pd.DataFrame.from_dict(SyncRecords_dict) + df_Summaries = pd.DataFrame.from_dict(summaries_dict) + df_SyncRecords = pd.DataFrame.from_dict(syncrecords_dict) df_all = df_Summaries.merge(df_SyncRecords, left_on=['Site', 'Month', 'Year'], right_on=['Site', 'Month', 'Year'], how='inner') fetchset = df_all.to_dict('index') @@ -358,42 +369,18 @@ def retrieve(self, request, SiteName=None, YearMonth=None): fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, SubmitHost AS SubmitHostSumm, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb, SubmitHost AS SubmitHostSync FROM VSyncRecords WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) - Summaries_dict = { - "Site": [], - "Month": [], - "Year": [], - "SubmitHostSumm": [], - "RecordCountPublished": [], - "RecordStart": [], - "RecordEnd": [] - } - - SyncRecords_dict = { - "Site": [], - "Month": [], - "Year": [], - "SubmitHostSync": [], - "RecordCountInDb": [] - } + summaries_dict = summaries_dict_standard.copy() + syncrecords_dict = syncrecords_dict_standard.copy() for row in fetchset_Summaries: - Summaries_dict["Site"] = Summaries_dict.get("Site") + [row.Site] - Summaries_dict["Month"] = Summaries_dict.get("Month") + [row.Month] - Summaries_dict["Year"] = Summaries_dict.get("Year") + [row.Year] - Summaries_dict["SubmitHostSumm"] = Summaries_dict.get("SubmitHostSumm") + [row.SubmitHostSumm] - Summaries_dict["RecordCountPublished"] = Summaries_dict.get("RecordCountPublished") + [row.RecordCountPublished] - Summaries_dict["RecordStart"] = Summaries_dict.get("RecordStart") + [row.RecordStart] - Summaries_dict["RecordEnd"] = Summaries_dict.get("RecordEnd") + [row.RecordEnd] - + summaries_dict = fill_summaries_dict(summaries_dict, row) + summaries_dict = correct_dict(summaries_dict) for row in fetchset_SyncRecords: - SyncRecords_dict["Site"] = SyncRecords_dict.get("Site") + [row.Site] - SyncRecords_dict["Month"] = SyncRecords_dict.get("Month") + [row.Month] - SyncRecords_dict["Year"] = SyncRecords_dict.get("Year") + [row.Year] - SyncRecords_dict["SubmitHostSync"] = SyncRecords_dict.get("SubmitHostSync") + [row.SubmitHostSync] - SyncRecords_dict["RecordCountInDb"] = SyncRecords_dict.get("RecordCountInDb") + [row.RecordCountInDb] - - df_Summaries = pd.DataFrame.from_dict(Summaries_dict) - df_SyncRecords = pd.DataFrame.from_dict(SyncRecords_dict) + syncrecords_dict = fill_syncrecords_dict(syncrecords_dict, row) + syncrecords_dict = correct_dict(syncrecords_dict) + + df_Summaries = pd.DataFrame.from_dict(summaries_dict) + df_SyncRecords = pd.DataFrame.from_dict(syncrecords_dict) df_Summaries.dropna(inplace=True) df_SyncRecords.dropna(inplace=True) From d8c409ed26d4330a2e93e96a2700527414a9d266 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 14:24:57 +0000 Subject: [PATCH 072/183] Bring functions outside class to avoid duplication. --- monitoring/publishing/views.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 37a1608..db10f1f 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -78,6 +78,15 @@ def correct_dict(inpDict): return inpDict +# Combine Year and Month into one string (display purposes) +def get_year_month_str(year, month): + year_string = str(year) + month_string = str(month) + if len(month_string) == 1: + month_string = '0' + month_string + return year_string + '-' + month_string + + class GridSiteViewSet(viewsets.ReadOnlyModelViewSet): queryset = GridSite.objects.all() serializer_class = GridSiteSerializer @@ -171,14 +180,6 @@ def get_template_names(self): elif self.action == 'retrieve': return ['gridsync_singlesite.html'] - # Combine Year and Month into one string (display purposes) - def get_year_month_string(self, year, month): - year_string = str(year) - month_string = str(month) - if len(month_string) == 1: - month_string = '0' + month_string - return year_string + '-' + month_string - def list(self, request): last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] n_sites = GridSiteSync.objects.values('SiteName').distinct().count() @@ -231,7 +232,7 @@ def list(self, request): 'RecordCountInDb': f.get("RecordCountInDb"), 'SyncStatus': f.get("SyncStatus"), }, - YearMonth=self.get_year_month_string(f.get("Year"), f.get("Month")), + YearMonth=get_year_month_str(f.get("Year"), f.get("Month")), SiteName=f.get("Site"), Month=f.get("Month"), Year=f.get("Year"), @@ -299,7 +300,7 @@ def retrieve(self, request, SiteName=None): 'RecordCountInDb': f.get("RecordCountInDb"), 'SyncStatus': f.get("SyncStatus"), }, - YearMonth=self.get_year_month_string(f.get("Year"), f.get("Month")), + YearMonth=get_year_month_str(f.get("Year"), f.get("Month")), SiteName=f.get("Site"), Month=f.get("Month"), Year=f.get("Year"), @@ -390,13 +391,6 @@ def retrieve(self, request, SiteName=None, YearMonth=None): # This is to list only data for one month GridSiteSyncSubmitH.objects.all().delete() - def get_year_month_string(year, month): - year_string = str(year) - month_string = str(month) - if len(month_string) == 1: - month_string = '0' + month_string - return year_string + '-' + month_string - for f in fetchset.values(): GridSiteSyncSubmitH.objects.update_or_create( defaults={ @@ -406,7 +400,7 @@ def get_year_month_string(year, month): 'RecordCountInDb': f.get("RecordCountInDb"), }, SiteName=f.get("Site"), - YearMonth=get_year_month_string(f.get("Year"), f.get("Month")), + YearMonth=get_year_month_str(f.get("Year"), f.get("Month")), Month=f.get("Month"), Year=f.get("Year"), SubmitHost=f.get("SubmitHostSumm"), From 0761c70f3f7ea0d497b9a69e4d7300bf7790af77 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 14:29:54 +0000 Subject: [PATCH 073/183] Refactor to reduce line length. --- monitoring/publishing/views.py | 49 +++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index db10f1f..f12b852 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -99,7 +99,10 @@ def list(self, request): if last_fetched is None or (last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20))): fetchset = VSuperSummaries.objects.using('grid').raw("SELECT Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries WHERE Year=2019 GROUP BY 1;") for f in fetchset: - GridSite.objects.update_or_create(defaults={'updated': f.LatestPublish}, SiteName=f.Site) + GridSite.objects.update_or_create( + defaults={'updated': f.LatestPublish}, + SiteName=f.Site + ) else: print('No need to update') @@ -138,7 +141,10 @@ def retrieve(self, request, pk=None): print('Out of date') fetchset = VSuperSummaries.objects.using('grid').raw("SELECT Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries WHERE Year=2019 GROUP BY 1;") for f in fetchset: - GridSite.objects.update_or_create(defaults={'updated': f.LatestPublish}, SiteName=f.Site) + GridSite.objects.update_or_create( + defaults={'updated': f.LatestPublish}, + SiteName=f.Site + ) else: print('No need to update') @@ -208,7 +214,12 @@ def list(self, request): # Merge data from VSuperSummaries and VSyncRecords into one df df_Summaries = pd.DataFrame.from_dict(summaries_dict) df_SyncRecords = pd.DataFrame.from_dict(syncrecords_dict) - df_all = df_Summaries.merge(df_SyncRecords, left_on=['Site', 'Month', 'Year'], right_on=['Site', 'Month', 'Year'], how='inner') + df_all = df_Summaries.merge( + df_SyncRecords, + left_on=['Site', 'Month', 'Year'], + right_on=['Site', 'Month', 'Year'], + how='inner' + ) fetchset = df_all.to_dict('index') # Delete all data if table not empty (as this function lists all sites) @@ -275,7 +286,12 @@ def retrieve(self, request, SiteName=None): df_Summaries = pd.DataFrame.from_dict(summaries_dict) df_SyncRecords = pd.DataFrame.from_dict(syncrecords_dict) - df_all = df_Summaries.merge(df_SyncRecords, left_on=['Site', 'Month', 'Year'], right_on=['Site', 'Month', 'Year'], how='inner') + df_all = df_Summaries.merge( + df_SyncRecords, + left_on=['Site', 'Month', 'Year'], + right_on=['Site', 'Month', 'Year'], + how='inner' + ) fetchset = df_all.to_dict('index') # Ensure we list only the data for one site @@ -385,7 +401,12 @@ def retrieve(self, request, SiteName=None, YearMonth=None): df_Summaries.dropna(inplace=True) df_SyncRecords.dropna(inplace=True) - df_all = df_Summaries.merge(df_SyncRecords, left_on=['Site', 'Month', 'Year', 'SubmitHostSumm'], right_on=['Site', 'Month', 'Year', 'SubmitHostSync'], how='outer') + df_all = df_Summaries.merge( + df_SyncRecords, + left_on=['Site', 'Month', 'Year', 'SubmitHostSumm'], + right_on=['Site', 'Month', 'Year', 'SubmitHostSync'], + how='outer' + ) fetchset = df_all.to_dict('index') # This is to list only data for one month @@ -430,7 +451,14 @@ def list(self, request): print('Out of date') fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID) as VMs, CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") for f in fetchset: - CloudSite.objects.update_or_create(defaults={'Vms': f.VMs, 'Script': f.CloudType, 'updated': f.UpdateTime}, SiteName=f.SiteName) + CloudSite.objects.update_or_create( + defaults={ + 'Vms': f.VMs, + 'Script': f.CloudType, + 'updated': f.UpdateTime + }, + SiteName=f.SiteName + ) else: print('No need to update') @@ -450,7 +478,14 @@ def retrieve(self, request, pk=None): print('Out of date') fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID) as VMs, CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") for f in fetchset: - CloudSite.objects.update_or_create(defaults={'Vms': f.VMs, 'Script': f.CloudType, 'updated': f.UpdateTime}, SiteName=f.SiteName) + CloudSite.objects.update_or_create( + defaults={ + 'Vms': f.VMs, + 'Script': f.CloudType, + 'updated': f.UpdateTime + }, + SiteName=f.SiteName + ) else: print('No need to update') From 2ac746214075c06ca0087f4f6c32e2f33ea9019e Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 14:33:46 +0000 Subject: [PATCH 074/183] Merge if statements to ensure clarity. --- monitoring/publishing/views.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index f12b852..19b8b8a 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -296,9 +296,8 @@ def retrieve(self, request, SiteName=None): # Ensure we list only the data for one site first_row = GridSiteSync.objects.first() - if first_row is not None: - if first_row.SiteName != SiteName: - GridSiteSync.objects.all().delete() + if hasattr(first_row, "SiteName") and first_row.SiteName != SiteName: + GridSiteSync.objects.all().delete() for f in fetchset.values(): rel_diff1 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountInDb")) From 22d1383ec5fc14e46762b9c068c3b4f655ca685b Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 16:03:28 +0000 Subject: [PATCH 075/183] Rearrange SQL queries to reduce line length. --- monitoring/publishing/views.py | 142 ++++++++++++++++++++++++++++++--- 1 file changed, 133 insertions(+), 9 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 19b8b8a..9a55560 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -97,7 +97,16 @@ def list(self, request): if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) if last_fetched is None or (last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20))): - fetchset = VSuperSummaries.objects.using('grid').raw("SELECT Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries WHERE Year=2019 GROUP BY 1;") + sql_query = """ + SELECT + Site, + max(LatestEndTime) AS LatestPublish + FROM VSuperSummaries + WHERE Year=2019 + GROUP BY 1; + """ + fetchset = VSuperSummaries.objects.using('grid').raw(sql_query) + for f in fetchset: GridSite.objects.update_or_create( defaults={'updated': f.LatestPublish}, @@ -139,7 +148,16 @@ def retrieve(self, request, pk=None): print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): print('Out of date') - fetchset = VSuperSummaries.objects.using('grid').raw("SELECT Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries WHERE Year=2019 GROUP BY 1;") + sql_query = """ + SELECT + Site, + max(LatestEndTime) AS LatestPublish + FROM VSuperSummaries + WHERE Year=2019 + GROUP BY 1; + """ + fetchset = VSuperSummaries.objects.using('grid').raw(sql_query) + for f in fetchset: GridSite.objects.update_or_create( defaults={'updated': f.LatestPublish}, @@ -196,8 +214,33 @@ def list(self, request): print('Out of date') # The condition on EarliestEndTime and LatestEndTime is necessary to avoid error by pytz because of dates like '00-00-00' - fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE EarliestEndTime>'1900-01-01' AND LatestEndTime>'1900-01-01' GROUP BY Site, Year, Month;") - fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords GROUP BY Site, Year, Month") + sql_query_summaries = """ + SELECT + Site, + Month, Year, + SUM(NumberOfJobs) AS RecordCountPublished, + MIN(EarliestEndTime) AS RecordStart, + MAX(LatestEndTime) AS RecordEnd + FROM VSuperSummaries + WHERE + EarliestEndTime>'1900-01-01' AND + LatestEndTime>'1900-01-01' + GROUP BY + Site, Year, Month; + """ + fetchset_Summaries = VSuperSummaries.objects.using('apel').raw(sql_query_summaries) + + sql_query_syncrec = """ + SELECT + Site, + Month, + Year, + SUM(NumberOfJobs) AS RecordCountInDb + FROM VSyncRecords + GROUP BY + Site, Year, Month; + """ + fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw(sql_query_syncrec) # Create empty dicts that will become dfs to be combined summaries_dict = summaries_dict_standard.copy() @@ -271,8 +314,36 @@ def retrieve(self, request, SiteName=None): print('Out of date') # The condition on EarliestEndTime and LatestEndTime is necessary to avoid error by pytz because of dates like '00-00-00' - fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND EarliestEndTime>'1900-01-01' AND LatestEndTime>'1900-01-01'GROUP BY Site, Month, Year;".format(SiteName)) - fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords WHERE Site='{}' GROUP BY Site, Month, Year;".format(SiteName)) + sql_query_summaries = """ + SELECT + Site, + Month, + Year, + SUM(NumberOfJobs) AS RecordCountPublished, + MIN(EarliestEndTime) AS RecordStart, + MAX(LatestEndTime) AS RecordEnd + FROM VSuperSummaries + WHERE + Site='{}' AND + EarliestEndTime>'1900-01-01' AND + LatestEndTime>'1900-01-01' + GROUP BY + Site, Year, Month; + """.format(SiteName) + fetchset_Summaries = VSuperSummaries.objects.using('apel').raw(sql_query_summaries) + + sql_query_syncrecords = """ + SELECT + Site, + Month, + Year, + SUM(NumberOfJobs) AS RecordCountInDb + FROM VSyncRecords + WHERE Site='{}' + GROUP BY + Site, Year, Month; + """.format(SiteName) + fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw(sql_query_syncrecords) summaries_dict = summaries_dict_standard.copy() syncrecords_dict = syncrecords_dict_standard.copy() @@ -382,8 +453,39 @@ def retrieve(self, request, SiteName=None, YearMonth=None): if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or (sitename_in_table != SiteName) or (yearmonth_in_table != YearMonth): print('Out of date') - fetchset_Summaries = VSuperSummaries.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountPublished, SubmitHost AS SubmitHostSumm, MIN(EarliestEndTime) AS RecordStart, MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) - fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw("SELECT Site, Month, Year, SUM(NumberOfJobs) AS RecordCountInDb, SubmitHost AS SubmitHostSync FROM VSyncRecords WHERE Site='{}' AND Month='{}' AND Year='{}' GROUP BY SubmitHost;".format(SiteName, Month, Year)) + sql_query_summaries = """ + SELECT + Site, + Month, + Year, + SUM(NumberOfJobs) AS RecordCountPublished, + SubmitHost AS SubmitHostSumm, + MIN(EarliestEndTime) AS RecordStart, + MAX(LatestEndTime) AS RecordEnd + FROM VSuperSummaries + WHERE + Site='{}' AND + Month='{}' AND + Year='{}' + GROUP BY SubmitHost; + """.format(SiteName, Month, Year) + fetchset_Summaries = VSuperSummaries.objects.using('apel').raw(sql_query_summaries) + + sql_query_syncrecords = """ + SELECT + Site, + Month, + Year, + SUM(NumberOfJobs) AS RecordCountInDb, + SubmitHost AS SubmitHostSync + FROM VSyncRecords + WHERE + Site='{}' AND + Month='{}' AND + Year='{}' + GROUP BY SubmitHost; + """.format(SiteName, Month, Year) + fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw(sql_query_syncrecords) summaries_dict = summaries_dict_standard.copy() syncrecords_dict = syncrecords_dict_standard.copy() @@ -448,7 +550,29 @@ def list(self, request): print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) if last_fetched is None or (last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20))): print('Out of date') - fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID) as VMs, CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") + + sql_query = """ + SELECT + b.SiteName, + COUNT(DISTINCT VMUUID) as VMs, + CloudType, + b.UpdateTime + FROM( + SELECT + SiteName, + MAX(UpdateTime) AS latest + FROM VAnonCloudRecords + WHERE UpdateTime>'2018-07-25' + GROUP BY SiteName + ) + AS a + INNER JOIN VAnonCloudRecords + AS b + ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest + GROUP BY SiteName; + """ + fetchset = VAnonCloudRecord.objects.using('cloud').raw(sql_query) + for f in fetchset: CloudSite.objects.update_or_create( defaults={ From 5ca8c666f861dd8ca86ca094216e196a185d8f70 Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 16:55:16 +0000 Subject: [PATCH 076/183] Create separate function for sync_status to reduce duplication. --- monitoring/publishing/views.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 9a55560..88e6694 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -78,6 +78,18 @@ def correct_dict(inpDict): return inpDict +def determine_sync_status(f): + RecordCountPublished = f.get("RecordCountPublished") + RecordCountInDb = f.get("RecordCountInDb") + rel_diff1 = abs(RecordCountPublished - RecordCountInDb)/RecordCountInDb + rel_diff2 = abs(RecordCountPublished - RecordCountInDb)/RecordCountPublished + if rel_diff1 < 0.01 or rel_diff2 < 0.01: + syncstatus = 'OK' + else: + syncstatus = 'Error' + return syncstatus + + # Combine Year and Month into one string (display purposes) def get_year_month_str(year, month): year_string = str(year) @@ -270,12 +282,7 @@ def list(self, request): # Determine SyncStatus based on the difference between records published and in db for f in fetchset.values(): - rel_diff1 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountInDb")) - rel_diff2 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountPublished")) - if rel_diff1 < 0.01 or rel_diff2 < 0.01: - f['SyncStatus'] = 'OK' - else: - f['SyncStatus'] = 'Error' + f['SyncStatus'] = determine_sync_status(f) # Combined primary keys outside the default dict GridSiteSync.objects.update_or_create( @@ -371,12 +378,7 @@ def retrieve(self, request, SiteName=None): GridSiteSync.objects.all().delete() for f in fetchset.values(): - rel_diff1 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountInDb")) - rel_diff2 = abs(f.get("RecordCountPublished") - f.get("RecordCountInDb"))/(f.get("RecordCountPublished")) - if rel_diff1 <= 0.01 or rel_diff2 <= 0.01: - f['SyncStatus'] = 'OK' - else: - f['SyncStatus'] = 'Error' + f['SyncStatus'] = determine_sync_status(f) GridSiteSync.objects.update_or_create( defaults={ From 5799dbb5dd6bd6575350100466c5dccea039700e Mon Sep 17 00:00:00 2001 From: Letizia Date: Fri, 3 Feb 2023 17:29:45 +0000 Subject: [PATCH 077/183] Create separate function foupdating stdout and returncode. --- monitoring/publishing/views.py | 40 +++++++++++++++------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 88e6694..20c29a0 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -45,6 +45,22 @@ } +def update_dict_stdout_and_returncode(single_dict, date): + diff = datetime.today() - date + date = date.strftime("%Y-%m-%d") + + if diff <= timedelta(days=7): + single_dict['returncode'] = 0 + single_dict['stdout'] = "OK [ last published %s days ago: %s ]" % (diff.days, date) + elif diff > timedelta(days=7): + single_dict['returncode'] = 1 + single_dict['stdout'] = "WARNING [ last published %s days ago: %s ]" % (diff.days, date) + else: + single_dict['returncode'] = 3 + single_dict['stdout'] = "UNKNOWN" + return single_dict + + def fill_summaries_dict(inpDict, row): inpDict["Site"] = inpDict.get("Site") + [row.Site] inpDict["Month"] = inpDict.get("Month") + [row.Month] @@ -132,17 +148,7 @@ def list(self, request): for single_dict in response.data: date = single_dict.get('updated').replace(tzinfo=None) - - diff = datetime.today() - date - if diff <= timedelta(days=7): - single_dict['returncode'] = 0 - single_dict['stdout'] = "OK [ last published %s days ago: %s ]" % (diff.days, date.strftime("%Y-%m-%d")) - elif diff > timedelta(days=7): - single_dict['returncode'] = 1 - single_dict['stdout'] = "WARNING [ last published %s days ago: %s ]" % (diff.days, date.strftime("%Y-%m-%d")) - else: - single_dict['returncode'] = 3 - single_dict['stdout'] = "UNKNOWN" + single_dict = update_dict_stdout_and_returncode(single_dict, date) final_response.append(single_dict) if type(request.accepted_renderer) is TemplateHTMLRenderer: @@ -189,17 +195,7 @@ def retrieve(self, request, pk=None): 'last_fetched': last_fetched } - diff = datetime.today() - date - if diff <= timedelta(days=7): - response.data['returncode'] = 0 - response.data['stdout'] = "OK [ last published %s days ago: %s ]" % (diff.days, date.strftime("%Y-%m-%d")) - elif diff > timedelta(days=7): - response.data['returncode'] = 1 - response.data['stdout'] = "WARNING [ last published %s days ago: %s ]" % (diff.days, date.strftime("%Y-%m-%d")) - else: - response.data['returncode'] = 3 - response.data['stdout'] = "UNKNOWN" - + response.data = update_dict_stdout_and_returncode(response.data, date) return response From 12b166eb1f53b6fed843506abc124114c993fd1e Mon Sep 17 00:00:00 2001 From: Letizia Date: Tue, 7 Feb 2023 13:21:20 +0000 Subject: [PATCH 078/183] Refactor to reduce length of lines. --- monitoring/publishing/views.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 20c29a0..e82538d 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -62,12 +62,19 @@ def update_dict_stdout_and_returncode(single_dict, date): def fill_summaries_dict(inpDict, row): - inpDict["Site"] = inpDict.get("Site") + [row.Site] - inpDict["Month"] = inpDict.get("Month") + [row.Month] - inpDict["Year"] = inpDict.get("Year") + [row.Year] - inpDict["RecordCountPublished"] = inpDict.get("RecordCountPublished") + [row.RecordCountPublished] - inpDict["RecordStart"] = inpDict.get("RecordStart") + [row.RecordStart] - inpDict["RecordEnd"] = inpDict.get("RecordEnd") + [row.RecordEnd] + + fields_to_update_and_value_to_add = { + "Site": row.Site, + "Month": row.Month, + "Year": row.Year, + "RecordCountPublished": row.RecordCountPublished, + "RecordStart": row.RecordStart, + "RecordEnd": row.RecordEnd, + } + + for field, value in fields_to_update_and_value_to_add.items(): + inpDict[field] = inpDict.get(field) + [value] + if hasattr(row, "SubmitHostSumm"): inpDict["SubmitHostSumm"] = inpDict.get("SubmitHostSumm") + [row.SubmitHostSumm] @@ -440,7 +447,7 @@ def retrieve(self, request, SiteName=None, YearMonth=None): sitename_in_table = None yearmonth_in_table = None - # This is to ensure the data is updated when changing (or clicking on another) month + # This is to ensure the data is updated when changing month if GridSiteSyncSubmitH.objects.count() > 0: row_1 = GridSiteSyncSubmitH.objects.filter()[:1].get() sitename_in_table = row_1.SiteName From b03791d662aa6750b01e0f99dbed3a489dced564 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 22 Nov 2023 11:19:01 +0000 Subject: [PATCH 079/183] Add missing objects that weren't created/imported --- monitoring/publishing/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index e82538d..fd59689 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -4,6 +4,7 @@ from datetime import datetime, timedelta from django.db.models import Max +from django.shortcuts import get_object_or_404 import pandas as pd from rest_framework import viewsets, generics @@ -432,6 +433,7 @@ class GridSiteSyncSubmitHViewSet(MultipleFieldLookupMixin, viewsets.ReadOnlyMode template_name = 'gridsync_submithost.html' def list(self, request): + last_fetched = GridSiteSyncSubmitH.objects.aggregate(Max('fetched'))['fetched__max'] response = super(GridSiteSyncSubmitHViewSet, self).list(request) response.data = { 'submisthosts': response.data, From ee92a54666db1a896a1e24409b3f2acddf5b8ef9 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 22 Nov 2023 11:20:37 +0000 Subject: [PATCH 080/183] Correct reference to grid database from settings --- monitoring/publishing/views.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index fd59689..9039c7c 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -244,7 +244,7 @@ def list(self, request): GROUP BY Site, Year, Month; """ - fetchset_Summaries = VSuperSummaries.objects.using('apel').raw(sql_query_summaries) + fetchset_Summaries = VSuperSummaries.objects.using('grid').raw(sql_query_summaries) sql_query_syncrec = """ SELECT @@ -256,7 +256,7 @@ def list(self, request): GROUP BY Site, Year, Month; """ - fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw(sql_query_syncrec) + fetchset_SyncRecords = VSyncRecords.objects.using('grid').raw(sql_query_syncrec) # Create empty dicts that will become dfs to be combined summaries_dict = summaries_dict_standard.copy() @@ -341,7 +341,7 @@ def retrieve(self, request, SiteName=None): GROUP BY Site, Year, Month; """.format(SiteName) - fetchset_Summaries = VSuperSummaries.objects.using('apel').raw(sql_query_summaries) + fetchset_Summaries = VSuperSummaries.objects.using('grid').raw(sql_query_summaries) sql_query_syncrecords = """ SELECT @@ -354,7 +354,7 @@ def retrieve(self, request, SiteName=None): GROUP BY Site, Year, Month; """.format(SiteName) - fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw(sql_query_syncrecords) + fetchset_SyncRecords = VSyncRecords.objects.using('grid').raw(sql_query_syncrecords) summaries_dict = summaries_dict_standard.copy() syncrecords_dict = syncrecords_dict_standard.copy() @@ -476,7 +476,7 @@ def retrieve(self, request, SiteName=None, YearMonth=None): Year='{}' GROUP BY SubmitHost; """.format(SiteName, Month, Year) - fetchset_Summaries = VSuperSummaries.objects.using('apel').raw(sql_query_summaries) + fetchset_Summaries = VSuperSummaries.objects.using('grid').raw(sql_query_summaries) sql_query_syncrecords = """ SELECT @@ -492,7 +492,7 @@ def retrieve(self, request, SiteName=None, YearMonth=None): Year='{}' GROUP BY SubmitHost; """.format(SiteName, Month, Year) - fetchset_SyncRecords = VSyncRecords.objects.using('apel').raw(sql_query_syncrecords) + fetchset_SyncRecords = VSyncRecords.objects.using('grid').raw(sql_query_syncrecords) summaries_dict = summaries_dict_standard.copy() syncrecords_dict = syncrecords_dict_standard.copy() From 3b3453db70d6859f03cdccc07be8878b0f1aac1f Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 22 Nov 2023 11:56:47 +0000 Subject: [PATCH 081/183] Automate venv activation Putting venv activation in the wsgi file means that Apache can invoke the venv and it doesn't need to be done manually. --- monitoring/wsgi.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/monitoring/wsgi.py b/monitoring/wsgi.py index ed72d1b..5fa221b 100644 --- a/monitoring/wsgi.py +++ b/monitoring/wsgi.py @@ -11,6 +11,12 @@ from django.core.wsgi import get_wsgi_application + +# Activate virtualenv +activate_path = os.path.expanduser("/usr/share/DJANGO_MONITORING_APP/venv/bin/activate_this.py") +with open(activate_path) as act: + exec(act.read(), dict(__file__=activate_path)) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "monitoring.settings") application = get_wsgi_application() From 9d647a609a9e60da18678c5c95a3dcc8300dddd6 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 22 Nov 2023 12:48:45 +0000 Subject: [PATCH 082/183] Tidy up settings --- monitoring/settings.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/monitoring/settings.py b/monitoring/settings.py index 5d3d44f..122180c 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -25,7 +25,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] +ALLOWED_HOSTS = [] # Application definition @@ -91,17 +91,19 @@ }, 'grid': { 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'localhost', + 'HOST': '', 'PORT': '3306', - 'NAME': 'django_test_grid', - 'USER': 'root', + 'NAME': '', + 'USER': '', + 'PASSWORD': '', }, 'cloud': { 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'localhost', + 'HOST': '', 'PORT': '3306', - 'NAME': 'django_test', - 'USER': 'root', + 'NAME': '', + 'USER': '', + 'PASSWORD': '', }, } From bd77d79f958a734fd364c5c97314d2992bf62f98 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 22 Nov 2023 15:14:56 +0000 Subject: [PATCH 083/183] Change sync status to match old pages --- monitoring/publishing/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 9039c7c..8e2c2cf 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -108,9 +108,9 @@ def determine_sync_status(f): rel_diff1 = abs(RecordCountPublished - RecordCountInDb)/RecordCountInDb rel_diff2 = abs(RecordCountPublished - RecordCountInDb)/RecordCountPublished if rel_diff1 < 0.01 or rel_diff2 < 0.01: - syncstatus = 'OK' + syncstatus = "OK" else: - syncstatus = 'Error' + syncstatus = "ERROR [ Please use the Gap Publisher to synchronise this dataset]" return syncstatus From 4b2902c044bff38c8e11c1c996680949d3595591 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 22 Nov 2023 17:49:56 +0000 Subject: [PATCH 084/183] Revert changes to availability app urls --- monitoring/availability/urls.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/monitoring/availability/urls.py b/monitoring/availability/urls.py index 12d45b8..3b2d575 100644 --- a/monitoring/availability/urls.py +++ b/monitoring/availability/urls.py @@ -1,8 +1,6 @@ from django.conf.urls import url -import sys -from monitoring.availability import views -sys.path.append('/usr/share/DJANGO_MONITORING_APP') +from monitoring.availability import views urlpatterns = [ url(r'^$', views.status), From afd03fa775a440296e6bb83bdf9397ea98859603 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 24 Jan 2024 16:11:32 +0000 Subject: [PATCH 085/183] Simplify availability app to just return "OK" 200 --- monitoring/availability/views.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/monitoring/availability/views.py b/monitoring/availability/views.py index 380575e..ba6a05b 100644 --- a/monitoring/availability/views.py +++ b/monitoring/availability/views.py @@ -1,15 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import time - from rest_framework.decorators import api_view from rest_framework.response import Response @api_view() def status(requst): - if int(time.time()) % 2: - return Response("Everything OK") - else: - return Response("Everything NOT ok.") + return Response("OK", status=200) From f933dd72051669d048ae1e53965b410ab570ef89 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 24 Jan 2024 16:15:48 +0000 Subject: [PATCH 086/183] Remove year limitation on grid pub SQL query --- monitoring/publishing/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 8e2c2cf..26cd927 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -138,7 +138,6 @@ def list(self, request): Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries - WHERE Year=2019 GROUP BY 1; """ fetchset = VSuperSummaries.objects.using('grid').raw(sql_query) From 0c39eac1d4436df925ee72331808514670afe0ae Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 24 Jan 2024 16:47:02 +0000 Subject: [PATCH 087/183] Add basic HTML template for status page --- monitoring/availability/templates/status.html | 1 + monitoring/availability/views.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 monitoring/availability/templates/status.html diff --git a/monitoring/availability/templates/status.html b/monitoring/availability/templates/status.html new file mode 100644 index 0000000..4fae6b6 --- /dev/null +++ b/monitoring/availability/templates/status.html @@ -0,0 +1 @@ +{{ message }} diff --git a/monitoring/availability/views.py b/monitoring/availability/views.py index ba6a05b..eff3094 100644 --- a/monitoring/availability/views.py +++ b/monitoring/availability/views.py @@ -7,4 +7,4 @@ @api_view() def status(requst): - return Response("OK", status=200) + return Response({"message": "OK"}, status=200, template_name="status.html") From 4d9f151f49e4cd88d845057f7136c8eb07909c8a Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 24 Jan 2024 16:57:26 +0000 Subject: [PATCH 088/183] Add availability app to installed apps This will hopefully register the templates it contains --- monitoring/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/monitoring/settings.py b/monitoring/settings.py index 122180c..ac61109 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -39,6 +39,7 @@ 'django.contrib.staticfiles', 'rest_framework', 'monitoring.publishing', + 'monitoring.availability', ] REST_FRAMEWORK = { From 40a7c6d62726254814f3b76ac007dc5180d7c121 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 28 Feb 2024 10:13:58 +0000 Subject: [PATCH 089/183] Update cutoff date for cloud query --- monitoring/publishing/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 26cd927..debc0b1 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -568,7 +568,7 @@ def list(self, request): SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords - WHERE UpdateTime>'2018-07-25' + WHERE UpdateTime>'2023-01-01' GROUP BY SiteName ) AS a From 20857e512145cf51a2a19cf044f3b939916ce03f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSae126V=E2=80=9D?= <“saitejav2021@gmail.com”> Date: Mon, 28 Oct 2024 17:45:45 +0000 Subject: [PATCH 090/183] Add configuration file --- conf/common.cfg | 27 +++++++++++++++++++++++++++ monitoring/settings.py | 31 ++++++++++++++++++------------- 2 files changed, 45 insertions(+), 13 deletions(-) create mode 100644 conf/common.cfg diff --git a/conf/common.cfg b/conf/common.cfg new file mode 100644 index 0000000..f1ac589 --- /dev/null +++ b/conf/common.cfg @@ -0,0 +1,27 @@ +[db] +# type of database +backend_grid = django.db.backends.mysql +backend_cloud = django.db.backends.mysql + +# host with database +hostname_grid = localhost +hostname_cloud = localhost + +# port to connect to +port_grid = 3306 +port_cloud = 3306 + +# database name +name_grid = +name_cloud = + +# database user +username_grid = root +username_cloud = root + +# password for database +password_grid = +password_cloud = + +[common] +allowed_hosts = diff --git a/monitoring/settings.py b/monitoring/settings.py index ac61109..fb7b1ee 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -11,10 +11,15 @@ """ import os +import configparser # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +# Read configuration from the file +cp = configparser.ConfigParser() +file_path = os.path.join(BASE_DIR, 'conf', 'common.cfg') +cp.read(file_path) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ @@ -25,7 +30,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = cp.get('common', 'allowed_hosts').split(',') # Application definition @@ -91,20 +96,20 @@ 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), }, 'grid': { - 'ENGINE': 'django.db.backends.mysql', - 'HOST': '', - 'PORT': '3306', - 'NAME': '', - 'USER': '', - 'PASSWORD': '', + 'ENGINE': cp.get('db', 'backend_grid'), + 'HOST': cp.get('db', 'hostname_grid'), + 'PORT': cp.get('db', 'port_grid'), + 'NAME': cp.get('db', 'name_grid'), + 'USER': cp.get('db', 'username_grid'), + 'PASSWORD': cp.get('db', 'password_grid'), }, 'cloud': { - 'ENGINE': 'django.db.backends.mysql', - 'HOST': '', - 'PORT': '3306', - 'NAME': '', - 'USER': '', - 'PASSWORD': '', + 'ENGINE': cp.get('db', 'backend_cloud'), + 'HOST': cp.get('db', 'hostname_cloud'), + 'PORT': cp.get('db', 'port_cloud'), + 'NAME': cp.get('db', 'name_cloud'), + 'USER': cp.get('db', 'username_cloud'), + 'PASSWORD': cp.get('db', 'password_cloud'), }, } From f42d9a6bad2d79678c3e1b383b822928c707bd28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSae126V=E2=80=9D?= <“saitejav2021@gmail.com”> Date: Tue, 29 Oct 2024 12:37:20 +0000 Subject: [PATCH 091/183] Update code to have try and catch block and address comments Update the catch block to handle the OSError(s) --- conf/common.cfg | 27 --------------- monitoring/settings.ini | 29 ++++++++++++++++ monitoring/settings.py | 74 +++++++++++++++++++++++------------------ 3 files changed, 70 insertions(+), 60 deletions(-) delete mode 100644 conf/common.cfg create mode 100644 monitoring/settings.ini diff --git a/conf/common.cfg b/conf/common.cfg deleted file mode 100644 index f1ac589..0000000 --- a/conf/common.cfg +++ /dev/null @@ -1,27 +0,0 @@ -[db] -# type of database -backend_grid = django.db.backends.mysql -backend_cloud = django.db.backends.mysql - -# host with database -hostname_grid = localhost -hostname_cloud = localhost - -# port to connect to -port_grid = 3306 -port_cloud = 3306 - -# database name -name_grid = -name_cloud = - -# database user -username_grid = root -username_cloud = root - -# password for database -password_grid = -password_cloud = - -[common] -allowed_hosts = diff --git a/monitoring/settings.ini b/monitoring/settings.ini new file mode 100644 index 0000000..e1b12a5 --- /dev/null +++ b/monitoring/settings.ini @@ -0,0 +1,29 @@ +# This file will be parsed by settings.py. + +[common] +# `allowed_hosts` values should be comma separated list of hostnames (fqdn's) +allowed_hosts = + + +# Information about the database connection - grid +[db grid] +# type of database - refers to the Django db backend +backend = django.db.backends.mysql + +hostname = localhost +port = 3306 +name = +username = root +password = + + +# Information about the database connection - cloud +[db cloud] +# type of database refers to the Django db backend +backend = django.db.backends.mysql + +hostname = localhost +port = 3306 +name = +username = root +password = diff --git a/monitoring/settings.py b/monitoring/settings.py index fb7b1ee..791cc6b 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -12,15 +12,11 @@ import os import configparser +import sys # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -# Read configuration from the file -cp = configparser.ConfigParser() -file_path = os.path.join(BASE_DIR, 'conf', 'common.cfg') -cp.read(file_path) - # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ @@ -30,8 +26,47 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = cp.get('common', 'allowed_hosts').split(',') +try: + + # Read configuration from the file + cp = configparser.ConfigParser() + file_path = os.path.join(BASE_DIR, 'monitoring', 'settings.ini') + + if not os.path.isfile(file_path): + raise FileNotFoundError(f'Configuration file NOT found: %s' % file_path) + + cp.read(file_path) + + ALLOWED_HOSTS = cp.get('common', 'allowed_hosts').split(',') + + # Database + # https://docs.djangoproject.com/en/1.11/ref/settings/#databases + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + }, + 'grid': { + 'ENGINE': cp.get('db grid', 'backend'), + 'HOST': cp.get('db grid', 'hostname'), + 'PORT': cp.get('db grid', 'port'), + 'NAME': cp.get('db grid', 'name'), + 'USER': cp.get('db grid', 'username'), + 'PASSWORD': cp.get('db grid', 'password'), + }, + 'cloud': { + 'ENGINE': cp.get('db cloud', 'backend'), + 'HOST': cp.get('db cloud', 'hostname'), + 'PORT': cp.get('db cloud', 'port'), + 'NAME': cp.get('db cloud', 'name'), + 'USER': cp.get('db cloud', 'username'), + 'PASSWORD': cp.get('db cloud', 'password'), + }, + } +except (OSError, configparser.Error) as err: + print("Error in configuration file: %s" % err) + sys.exit(1) # Application definition @@ -87,33 +122,6 @@ WSGI_APPLICATION = 'monitoring.wsgi.application' -# Database -# https://docs.djangoproject.com/en/1.11/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - }, - 'grid': { - 'ENGINE': cp.get('db', 'backend_grid'), - 'HOST': cp.get('db', 'hostname_grid'), - 'PORT': cp.get('db', 'port_grid'), - 'NAME': cp.get('db', 'name_grid'), - 'USER': cp.get('db', 'username_grid'), - 'PASSWORD': cp.get('db', 'password_grid'), - }, - 'cloud': { - 'ENGINE': cp.get('db', 'backend_cloud'), - 'HOST': cp.get('db', 'hostname_cloud'), - 'PORT': cp.get('db', 'port_cloud'), - 'NAME': cp.get('db', 'name_cloud'), - 'USER': cp.get('db', 'username_cloud'), - 'PASSWORD': cp.get('db', 'password_cloud'), - }, -} - - # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators From ac19a35a5624a1b382872bb53e4fea7b61b7c200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSae126V=E2=80=9D?= <“saitejav2021@gmail.com”> Date: Tue, 29 Oct 2024 14:01:03 +0000 Subject: [PATCH 092/183] Rename section headers and fix the file path --- monitoring/settings.ini | 4 ++-- monitoring/settings.py | 30 +++++++++++++----------------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/monitoring/settings.ini b/monitoring/settings.ini index e1b12a5..33ac1ba 100644 --- a/monitoring/settings.ini +++ b/monitoring/settings.ini @@ -6,7 +6,7 @@ allowed_hosts = # Information about the database connection - grid -[db grid] +[db_grid] # type of database - refers to the Django db backend backend = django.db.backends.mysql @@ -18,7 +18,7 @@ password = # Information about the database connection - cloud -[db cloud] +[db_cloud] # type of database refers to the Django db backend backend = django.db.backends.mysql diff --git a/monitoring/settings.py b/monitoring/settings.py index 791cc6b..43ad745 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -31,10 +31,6 @@ # Read configuration from the file cp = configparser.ConfigParser() file_path = os.path.join(BASE_DIR, 'monitoring', 'settings.ini') - - if not os.path.isfile(file_path): - raise FileNotFoundError(f'Configuration file NOT found: %s' % file_path) - cp.read(file_path) ALLOWED_HOSTS = cp.get('common', 'allowed_hosts').split(',') @@ -47,24 +43,24 @@ 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), }, 'grid': { - 'ENGINE': cp.get('db grid', 'backend'), - 'HOST': cp.get('db grid', 'hostname'), - 'PORT': cp.get('db grid', 'port'), - 'NAME': cp.get('db grid', 'name'), - 'USER': cp.get('db grid', 'username'), - 'PASSWORD': cp.get('db grid', 'password'), + 'ENGINE': cp.get('db_grid', 'backend'), + 'HOST': cp.get('db_grid', 'hostname'), + 'PORT': cp.get('db_grid', 'port'), + 'NAME': cp.get('db_grid', 'name'), + 'USER': cp.get('db_grid', 'username'), + 'PASSWORD': cp.get('db_grid', 'password'), }, 'cloud': { - 'ENGINE': cp.get('db cloud', 'backend'), - 'HOST': cp.get('db cloud', 'hostname'), - 'PORT': cp.get('db cloud', 'port'), - 'NAME': cp.get('db cloud', 'name'), - 'USER': cp.get('db cloud', 'username'), - 'PASSWORD': cp.get('db cloud', 'password'), + 'ENGINE': cp.get('db_cloud', 'backend'), + 'HOST': cp.get('db_cloud', 'hostname'), + 'PORT': cp.get('db_cloud', 'port'), + 'NAME': cp.get('db_cloud', 'name'), + 'USER': cp.get('db_cloud', 'username'), + 'PASSWORD': cp.get('db_cloud', 'password'), }, } -except (OSError, configparser.Error) as err: +except (configparser.NoSectionError) as err: print("Error in configuration file: %s" % err) sys.exit(1) From b8f88b0e6452d227df1f2ea691bdccd142865592 Mon Sep 17 00:00:00 2001 From: Saiteja Reddy Vennapusa <127398882+Sae126V@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:00:36 +0000 Subject: [PATCH 093/183] Apply suggestions from code review Co-authored-by: Adrian Coveney <4836233+tofu-rocketry@users.noreply.github.com> --- monitoring/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/settings.py b/monitoring/settings.py index 43ad745..29dd4b6 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -10,8 +10,8 @@ https://docs.djangoproject.com/en/1.11/ref/settings/ """ -import os import configparser +import os import sys # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -61,7 +61,7 @@ } except (configparser.NoSectionError) as err: - print("Error in configuration file: %s" % err) + print("Error in configuration file. Check that file exists first: %s" % err) sys.exit(1) # Application definition From e1299ae43a9d4fee5ae649eececffdfd8209d8fb Mon Sep 17 00:00:00 2001 From: Saiteja Reddy Vennapusa <127398882+Sae126V@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:26:42 +0000 Subject: [PATCH 094/183] Update packages to work with py3.6 (#23) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update packages to work with py3.6 * Remove mysqlclient as it been replaced with pymsql --------- Co-authored-by: “Sae126V” <“saitejav2021@gmail.com”> --- docs/what_gets_installed.md | 16 ++++++++-------- requirements.txt | 10 ++++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/what_gets_installed.md b/docs/what_gets_installed.md index 0d43cce..381be5a 100644 --- a/docs/what_gets_installed.md +++ b/docs/what_gets_installed.md @@ -6,18 +6,18 @@ Following the config file that Aquilon uses, the following are the packages inst - python3-mod_wsgi (for apache to work with django) - python3-devel - gcc (needed for dependencies) -- mariadb. - +- mariadb +- tar. ## Packages installed within the venv Within venv, the following are installed through pip: -- djangorestframework (3.11.2) -- pymysql (needed for mariadb to work) -- pandas (needed by the app) -- django (2.1.*). +- djangorestframework (3.15.1) +- pymysql (1.0.2) (needed for mariadb to work) +- pandas (1.1.5) (needed by the app) +- django (3.1.14) +- pytz (2024.2). Note that when the version of the packages is specified, the app would not work with a different version (due to dependencies conflicts). -Also, as long as mariadb is installed (both client and server), it is not necessary to install mysqlclient (at least when the OS is Scientific Linux). Is is also important to note that different types of OS require different packages to be installed. -The above are the packages that allow the app to work on a scientific linux 7 machine. A Centos machine would require slightly different packages. +The above are the packages that allow the app to work on a Rocky8. diff --git a/requirements.txt b/requirements.txt index 54be296..581d42c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ -Django==2.2.28 -djangorestframework==3.11.2 -mysqlclient==1.3.9 -pytz==2019.3 +# Pin packages to support and work with py3.6. +Django==3.1.14 +djangorestframework==3.15.1 +pytz==2024.2 +PyMySQL==1.0.2 +pandas==1.1.5 From e244961b82aa83a5526aa1372a3ca389234314df Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 28 Feb 2024 16:34:10 +0000 Subject: [PATCH 095/183] Add styles sheets and fix references to them --- monitoring/publishing/static/style.css | 53 +++++ monitoring/publishing/static/stylesheet.css | 184 ++++++++++++++++++ .../publishing/templates/gridsites.html | 6 +- monitoring/publishing/templates/gridsync.html | 6 +- .../templates/gridsync_singlesite.html | 6 +- .../templates/gridsync_submithost.html | 6 +- 6 files changed, 253 insertions(+), 8 deletions(-) create mode 100644 monitoring/publishing/static/style.css create mode 100644 monitoring/publishing/static/stylesheet.css diff --git a/monitoring/publishing/static/style.css b/monitoring/publishing/static/style.css new file mode 100644 index 0000000..cdc2843 --- /dev/null +++ b/monitoring/publishing/static/style.css @@ -0,0 +1,53 @@ +/*- Menu Tabs E--------------------------- */ + + #tabsE { + float:left; + width:100%; + background:#333; + font-size:93%; + line-height:normal; + + } + #tabsE ul { + margin:0; + padding:10px 10px 0 50px; + list-style:none; + } + #tabsE li { + display:inline; + margin:0; + padding:0; + } + #tabsE a { + float:left; + background:url("tableftE.gif") no-repeat left top; + margin:0; + padding:0 0 0 4px; + text-decoration:none; + } + #tabsE a span { + float:left; + display:block; + background:url("tabrightE.gif") no-repeat right top; + padding:5px 15px 4px 6px; + color:#fff; + } + /* Commented Backslash Hack hides rule from IE5-Mac \*/ + #tabsE a span {float:none;} + /* End IE5-Mac hack */ + #tabsE a:hover span { + color:#FFF; + } + #tabsE a:hover { + background-position:0% -42px; + } + #tabsE a:hover span { + background-position:100% -42px; + } + + #tabsE #current a { + background-position:0% -42px; + } + #tabsE #current a span { + background-position:100% -42px; + } diff --git a/monitoring/publishing/static/stylesheet.css b/monitoring/publishing/static/stylesheet.css new file mode 100644 index 0000000..67b62dc --- /dev/null +++ b/monitoring/publishing/static/stylesheet.css @@ -0,0 +1,184 @@ +h12 { + font-size: 22px; + color: #336699; + background-color: #FFFFFF; + font-family: Arial, Helvetica, sans-serif; + font-weight: bold +} +h1 { + font-size: 1.2em; + color: #336699; + background-color: #FFFFFF; + font-family: Arial, Helvetica, sans-serif; +} +h2 { + font-size: 1em; +} + + +h6 { + align: right; + font-size: 0.6em; +} +body { + font-family: Arial, Helvetica, sans-serif; + background-color: #FFFFFF; + color: #000000; + font-size: .5px; + font-size: 14px; +} +th { + font-family: Arial, Helvetica, sans-serif; + font-size: 12px; + color: #009999; + background-color: #FFFFFF; + font-weight: bold + align: left; +} +a:link { + text-decoration: none; + color: #336699; + font-weight: bold; +} +a:active { + text-decoration: none; + color: #336699; + font-weight: bold; + background-color: #FFFFFF; +} +a:visited { + text-decoration: none; + color: #996699; + font-weight: bold; +} +a:hover { + color : #666666; + background-color: #CCCCCC; + font-weight: bold; +} +hr { + color: #666666; + background-color: #FFFFFF; +} +.outlined { + border: 1px solid #000000; +} +.navbar-title { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.8em; + color: #FFFFFF; + font-weight: bold; + + +} +.navbar { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.6em; + background-color: #FFFFFF; + +} +p { + font-family: Arial, Helvetica, sans-serif; + color: #000000; + font-size: 14px; + font-weight: normal; +} +li { + font-family: Arial, Helvetica, sans-serif; + font-size: 1em; + color: #111111; + list-style-type: square; + font-weight: normal; +} +.sidebar-orange { + background-color: #ffe5b2; + background-image: url(images/orange-globe.jpg); + background-position: right center; + background-repeat: no-repeat; + +} +.sidebar-green { + background-image: url(images/green-pulse.jpg); + background-repeat: repeat-x; + background-position: center bottom; + background-color: #dcf6de; +} +.sidebar-blue { + background-color: #dcf4f6; + background-image: url(images/blue-bars.jpg); + background-repeat: no-repeat; + background-position: right bottom; +} +.sidebar-pink { + background-color: #efdcf6; + background-image: url(images/pink-news.jpg); + background-repeat: no-repeat; + background-position: left bottom; +} +.note { + background-color: #E4E4E4; + border: 1px dotted #000000; + padding: 5px; +} +.tabletext { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.8em; + color: #000000; + background-color: #DDDDDD; +} +.tabletextwarning { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.8em; + color: #000000; + background-color: #FFFF00; +} +.tabletextok { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.8em; + color: #000000; + background-color: #00FF00; +} +.tabletexterror { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.8em; + color: #000000; + background-color: #FF0000; +} +.tabletextinfo { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.8em; + color: #000000; + background-color: #00CCFF; +} +.tableheader { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.8em; + color: #FFFFFF; + background-color: #000000; + font-weight: bold; + text-align: center; +} +.navbar-heading { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.8em; + background-color: #FFFFFF; + font-weight: bold; + color: #000000; +} +.feintoutlined { + border: 1px dotted #CCCCCC; +} +.newsHeader { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.9em; + background-color: #97A6CD; + font-weight: bold; +} +.newsBody { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.7em; + background-color: #ACBBDA; +} +.invisibleBorder { + border: thin dashed #EEEEEE; +} diff --git a/monitoring/publishing/templates/gridsites.html b/monitoring/publishing/templates/gridsites.html index 0ec21b0..0de8d17 100644 --- a/monitoring/publishing/templates/gridsites.html +++ b/monitoring/publishing/templates/gridsites.html @@ -1,11 +1,13 @@ +{% load static %} + APEL Publication Summary : {{ sites.0.name }} - - + +

    APEL Publication Test

    diff --git a/monitoring/publishing/templates/gridsync.html b/monitoring/publishing/templates/gridsync.html index 1b43e97..d7fd074 100644 --- a/monitoring/publishing/templates/gridsync.html +++ b/monitoring/publishing/templates/gridsync.html @@ -1,10 +1,12 @@ +{% load static %} + APEL Publication Summary - - + +

    APEL Synchronisation Test

    diff --git a/monitoring/publishing/templates/gridsync_singlesite.html b/monitoring/publishing/templates/gridsync_singlesite.html index 00f7aca..559807e 100644 --- a/monitoring/publishing/templates/gridsync_singlesite.html +++ b/monitoring/publishing/templates/gridsync_singlesite.html @@ -1,10 +1,12 @@ +{% load static %} + APEL Publication Summary - - + +

    APEL Synchronisation Test

    diff --git a/monitoring/publishing/templates/gridsync_submithost.html b/monitoring/publishing/templates/gridsync_submithost.html index a919ea6..89ceed0 100644 --- a/monitoring/publishing/templates/gridsync_submithost.html +++ b/monitoring/publishing/templates/gridsync_submithost.html @@ -1,10 +1,12 @@ +{% load static %} + APEL Publication Summary - - + +

    APEL Synchronisation Test

    From 7f8ee867e41fa6bd4a1e5a4558c0301e59d3e51c Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 14 Nov 2024 10:18:14 +0000 Subject: [PATCH 096/183] Add pre-commit config --- .pre-commit-config.yaml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..545b8f8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,38 @@ +# See https://pre-commit.com for more information +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 # Python 3.6 compatible + hooks: + # Python related checks + - id: check-ast + - id: check-builtin-literals + - id: check-docstring-first + - id: name-tests-test + name: Check unit tests start with 'test_' + args: ['--django'] + files: 'test/.*' + # Other checks + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-yaml + - id: debug-statements + - id: detect-private-key + - id: end-of-file-fixer + - id: mixed-line-ending + name: Force line endings to LF + args: ['--fix=lf'] + - id: trailing-whitespace + +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: python-check-mock-methods + - id: python-no-eval + - id: python-no-log-warn + - id: python-use-type-annotations + +# Pre-commit CI config, see https://pre-commit.ci/ +ci: + autofix_prs: false + autoupdate_schedule: quarterly From 5d7228ebea82d8dfe4427db4a28634191d8a5ff3 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 14 Nov 2024 10:20:35 +0000 Subject: [PATCH 097/183] Tidy whitespace, line endings and file endings --- docs/what_gets_installed.md | 2 +- monitoring/availability/templates/status.html | 2 +- monitoring/publishing/static/style.css | 2 +- monitoring/publishing/static/stylesheet.css | 10 +++++----- monitoring/publishing/templates/gridsites.html | 2 +- monitoring/publishing/templates/gridsync.html | 6 +++--- .../publishing/templates/gridsync_singlesite.html | 6 +++--- .../publishing/templates/gridsync_submithost.html | 4 ++-- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/what_gets_installed.md b/docs/what_gets_installed.md index 381be5a..ade3ecd 100644 --- a/docs/what_gets_installed.md +++ b/docs/what_gets_installed.md @@ -19,5 +19,5 @@ Within venv, the following are installed through pip: Note that when the version of the packages is specified, the app would not work with a different version (due to dependencies conflicts). -Is is also important to note that different types of OS require different packages to be installed. +Is is also important to note that different types of OS require different packages to be installed. The above are the packages that allow the app to work on a Rocky8. diff --git a/monitoring/availability/templates/status.html b/monitoring/availability/templates/status.html index 4fae6b6..9b504ca 100644 --- a/monitoring/availability/templates/status.html +++ b/monitoring/availability/templates/status.html @@ -1 +1 @@ -{{ message }} +{{ message }} diff --git a/monitoring/publishing/static/style.css b/monitoring/publishing/static/style.css index cdc2843..85a5cf9 100644 --- a/monitoring/publishing/static/style.css +++ b/monitoring/publishing/static/style.css @@ -43,7 +43,7 @@ } #tabsE a:hover span { background-position:100% -42px; - } + } #tabsE #current a { background-position:0% -42px; diff --git a/monitoring/publishing/static/stylesheet.css b/monitoring/publishing/static/stylesheet.css index 67b62dc..88f63af 100644 --- a/monitoring/publishing/static/stylesheet.css +++ b/monitoring/publishing/static/stylesheet.css @@ -32,27 +32,27 @@ th { font-size: 12px; color: #009999; background-color: #FFFFFF; - font-weight: bold + font-weight: bold align: left; } a:link { - text-decoration: none; + text-decoration: none; color: #336699; font-weight: bold; } a:active { - text-decoration: none; + text-decoration: none; color: #336699; font-weight: bold; background-color: #FFFFFF; } a:visited { - text-decoration: none; + text-decoration: none; color: #996699; font-weight: bold; } a:hover { - color : #666666; + color : #666666; background-color: #CCCCCC; font-weight: bold; } diff --git a/monitoring/publishing/templates/gridsites.html b/monitoring/publishing/templates/gridsites.html index 0de8d17..8abf32b 100644 --- a/monitoring/publishing/templates/gridsites.html +++ b/monitoring/publishing/templates/gridsites.html @@ -34,4 +34,4 @@

    APEL Publication Test

    - \ No newline at end of file + diff --git a/monitoring/publishing/templates/gridsync.html b/monitoring/publishing/templates/gridsync.html index d7fd074..7edccda 100644 --- a/monitoring/publishing/templates/gridsync.html +++ b/monitoring/publishing/templates/gridsync.html @@ -32,14 +32,14 @@

    APEL Synchronisation Test

    {% for record in records %} {{ record.SiteName }} - {{ record.YearMonth }} + {{ record.YearMonth }} {{ record.RecordStart }} {{ record.RecordEnd }} {{ record.RecordCountPublished }} {{ record.RecordCountInDb }} {{ record.SyncStatus }} - + {% endfor %} - \ No newline at end of file + diff --git a/monitoring/publishing/templates/gridsync_singlesite.html b/monitoring/publishing/templates/gridsync_singlesite.html index 559807e..f6e9dc5 100644 --- a/monitoring/publishing/templates/gridsync_singlesite.html +++ b/monitoring/publishing/templates/gridsync_singlesite.html @@ -30,14 +30,14 @@

    APEL Synchronisation Test

    {% for record in records %} - {{ record.YearMonth }} + {{ record.YearMonth }} {{ record.RecordStart }} {{ record.RecordEnd }} {{ record.RecordCountPublished }} {{ record.RecordCountInDb }} {{ record.SyncStatus }} - + {% endfor %} - \ No newline at end of file + diff --git a/monitoring/publishing/templates/gridsync_submithost.html b/monitoring/publishing/templates/gridsync_submithost.html index 89ceed0..6bb115e 100644 --- a/monitoring/publishing/templates/gridsync_submithost.html +++ b/monitoring/publishing/templates/gridsync_submithost.html @@ -30,7 +30,7 @@

    APEL Synchronisation Test

    {% for host in submisthosts %} - + {{ host.YearMonth }} {{ host.SubmitHost }} {{ host.RecordStart }} @@ -42,4 +42,4 @@

    APEL Synchronisation Test

    {% endfor %} - \ No newline at end of file + From 420075af15f4a06b915705c28cdf4b31720c4483 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 4 Dec 2024 10:33:31 +0000 Subject: [PATCH 098/183] Change DEBUG to False --- monitoring/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/settings.py b/monitoring/settings.py index 29dd4b6..724f7d9 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -24,7 +24,7 @@ SECRET_KEY = 'ge^fd9rf)htmxji8kf=jk8frh3=^11@^n=h14gu*fqt^0-lnr$' # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False try: From 64d78718f26d355ee076cadade54773213bf4c22 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 4 Dec 2024 10:34:21 +0000 Subject: [PATCH 099/183] Move SECRET_KEY to settings.ini for config man Move SECRET_KEY to be set from settings.ini so that configuration managment can set it from a secrets file. Also update link to docs for deployment checklist. --- monitoring/settings.ini | 3 +++ monitoring/settings.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/monitoring/settings.ini b/monitoring/settings.ini index 33ac1ba..222b25c 100644 --- a/monitoring/settings.ini +++ b/monitoring/settings.ini @@ -1,6 +1,9 @@ # This file will be parsed by settings.py. [common] +# A new key can be generated using django.core.management.utils.get_random_secret_key() +secret_key = + # `allowed_hosts` values should be comma separated list of hostnames (fqdn's) allowed_hosts = diff --git a/monitoring/settings.py b/monitoring/settings.py index 724f7d9..fcd1764 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -18,10 +18,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'ge^fd9rf)htmxji8kf=jk8frh3=^11@^n=h14gu*fqt^0-lnr$' +# See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False @@ -33,6 +30,9 @@ file_path = os.path.join(BASE_DIR, 'monitoring', 'settings.ini') cp.read(file_path) + # SECURITY WARNING: keep the secret key used in production secret! + SECRET_KEY = cp.get('common', 'secret_key') + ALLOWED_HOSTS = cp.get('common', 'allowed_hosts').split(',') # Database From 525e17f5bbcf70965d3c48bcc452ec36fc714f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CSae126V=E2=80=9D?= <“saitejav2021@gmail.com”> Date: Wed, 12 Feb 2025 11:10:40 +0000 Subject: [PATCH 100/183] Fix issue with ConfigParser interpolation by disabling - Some keys in SECRET_KEY can contain %, which configparser assumes are interpolated values --- monitoring/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/settings.py b/monitoring/settings.py index fcd1764..f427268 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -26,7 +26,7 @@ try: # Read configuration from the file - cp = configparser.ConfigParser() + cp = configparser.ConfigParser(interpolation=None) file_path = os.path.join(BASE_DIR, 'monitoring', 'settings.ini') cp.read(file_path) From ab0697ff026605ebe9c8f71798a872cc176e580e Mon Sep 17 00:00:00 2001 From: Sae126V Date: Wed, 26 Feb 2025 21:19:02 +0000 Subject: [PATCH 101/183] Remove unused imports in url.py --- monitoring/publishing/urls.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py index 7f8d4e4..8f47f56 100644 --- a/monitoring/publishing/urls.py +++ b/monitoring/publishing/urls.py @@ -1,5 +1,3 @@ -from django.conf.urls import include, url - from rest_framework import routers from monitoring.publishing import views From 92b6e55aad02f11f1ef2556a06c9da10b99bdc1a Mon Sep 17 00:00:00 2001 From: Sae126V Date: Wed, 26 Feb 2025 21:35:17 +0000 Subject: [PATCH 102/183] Update code to allow dot(s) in regex pattern - We need this currently because it is breaking the current code flow and returning an internal server error. - In the DB we have few sites with dot in their SiteName - Perhaps, Nice-To-Have is either from the codebase we restrict showing the sites with dot(Unnecessary load to fetch) or restrict the sites to make their way to DB. --- monitoring/publishing/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py index 8f47f56..21c78cd 100644 --- a/monitoring/publishing/urls.py +++ b/monitoring/publishing/urls.py @@ -11,12 +11,12 @@ urlpatterns = [ re_path( - r'^gridsync/(?P[a-zA-Z0-9-]+)/$', + r'^gridsync/(?P[a-zA-Z0-9.-]+)/$', views.GridSiteSyncViewSet.as_view({'get': 'retrieve'}), name='gridsync_singlesite' ), re_path( - r'^gridsync/(?P[a-zA-Z0-9-]+)/(?P[0-9-]+)/$', + r'^gridsync/(?P[a-zA-Z0-9.-]+)/(?P[0-9-]+)/$', views.GridSiteSyncSubmitHViewSet.as_view({'get': 'retrieve'}), name='gridsync_submithost' ), From 3b20da70d1b3d3bc2146c6e69159248ed7d859f1 Mon Sep 17 00:00:00 2001 From: Sae126V Date: Wed, 26 Feb 2025 22:22:50 +0000 Subject: [PATCH 103/183] Update code to use SiteName as a lookup_field - Removed the restriction on grid/ fetching the data with one year ONLY. - urls.py: We need this re_path() for grid and cloud to avoid getting the message saying "Could not resolve URL for hyperlinked relationship" - We need Django Rest Framework fetching the data based on the SiteName - We need this for couple of reasons 1) To avoid showing the data with the URL containing pk(As a Number like) in it 2) To fetch the data using SiteName as primary lookup field. --- monitoring/publishing/serializers.py | 10 ++++++++++ monitoring/publishing/urls.py | 10 ++++++++++ monitoring/publishing/views.py | 8 ++++---- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index de73fd1..24c9bc4 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -22,6 +22,11 @@ class Meta: 'updated' ) + # Sitename substitutes pk + lookup_field = 'SiteName' + extra_kwargs = { + 'url': {'view_name': 'gridsite-detail','lookup_field': 'SiteName'} + } class GridSiteSyncSerializer(serializers.HyperlinkedModelSerializer): # Override default format with None so that Python datetime is used as @@ -64,6 +69,11 @@ class Meta: 'updated' ) + # Sitename substitutes pk + lookup_field = 'SiteName' + extra_kwargs = { + 'url': {'view_name': 'cloudsite-detail', 'lookup_field': 'SiteName'} + } class GridSiteSyncSubmitHSerializer(serializers.HyperlinkedModelSerializer): # Override default format with None so that Python datetime is used as diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py index 21c78cd..f53981a 100644 --- a/monitoring/publishing/urls.py +++ b/monitoring/publishing/urls.py @@ -10,6 +10,16 @@ router.register(r'gridsync', views.GridSiteSyncSubmitHViewSet) urlpatterns = [ + re_path( + r'^cloud/(?P[a-zA-Z0-9.-]+)/$', + views.CloudSiteViewSet.as_view({'get': 'retrieve'}), + name='cloudsite-detail' + ), + re_path( + r'^grid/(?P[a-zA-Z0-9.-]+)/$', + views.GridSiteViewSet.as_view({'get': 'retrieve'}), + name='gridsite-detail' + ), re_path( r'^gridsync/(?P[a-zA-Z0-9.-]+)/$', views.GridSiteSyncViewSet.as_view({'get': 'retrieve'}), diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index debc0b1..c492f88 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -127,6 +127,7 @@ class GridSiteViewSet(viewsets.ReadOnlyModelViewSet): queryset = GridSite.objects.all() serializer_class = GridSiteSerializer template_name = 'gridsites.html' + lookup_field = 'SiteName' def list(self, request): last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max'] @@ -166,7 +167,7 @@ def list(self, request): return response - def retrieve(self, request, pk=None): + def retrieve(self, request, SiteName=None): last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max'] # If there's no data then last_fetched is None. if last_fetched is not None: @@ -178,7 +179,6 @@ def retrieve(self, request, pk=None): Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries - WHERE Year=2019 GROUP BY 1; """ fetchset = VSuperSummaries.objects.using('grid').raw(sql_query) @@ -313,7 +313,6 @@ def list(self, request): return response def retrieve(self, request, SiteName=None): - lookup_field = 'SiteName' last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] row_1 = GridSiteSync.objects.filter()[:1].get() n_sites = GridSiteSync.objects.values('SiteName').distinct().count() @@ -549,6 +548,7 @@ class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet): queryset = CloudSite.objects.all() serializer_class = CloudSiteSerializer template_name = 'cloudsites.html' + lookup_field = 'SiteName' def list(self, request): last_fetched = CloudSite.objects.aggregate(Max('fetched'))['fetched__max'] @@ -600,7 +600,7 @@ def list(self, request): } return response - def retrieve(self, request, pk=None): + def retrieve(self, request, SiteName=None): last_fetched = CloudSite.objects.aggregate(Max('fetched'))['fetched__max'] print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) if last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): From e336bcdd5058862622421b4de507f161ec35fd49 Mon Sep 17 00:00:00 2001 From: Sae126V Date: Wed, 26 Feb 2025 22:45:21 +0000 Subject: [PATCH 104/183] Update code to make gridsync URL work with the sitename(s) - Sometimes gridsync/ fetches the whole list again, ignoring the lookup_up field because we were using list() - We are using get_serializer() with `many=True` to handle complex data which is returning multiple objects with the single siteName. --- monitoring/publishing/serializers.py | 2 +- monitoring/publishing/urls.py | 3 +-- monitoring/publishing/views.py | 11 +++++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 24c9bc4..a4092d1 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -49,7 +49,7 @@ class Meta: # Sitename substitutes pk lookup_field = 'SiteName' extra_kwargs = { - 'url': {'lookup_field': 'SiteName'} + 'url': {'view_name': 'gridsitesync-detail', 'lookup_field': 'SiteName'} } diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py index f53981a..94c0d4b 100644 --- a/monitoring/publishing/urls.py +++ b/monitoring/publishing/urls.py @@ -7,7 +7,6 @@ router.register(r'cloud', views.CloudSiteViewSet) router.register(r'grid', views.GridSiteViewSet) router.register(r'gridsync', views.GridSiteSyncViewSet) -router.register(r'gridsync', views.GridSiteSyncSubmitHViewSet) urlpatterns = [ re_path( @@ -23,7 +22,7 @@ re_path( r'^gridsync/(?P[a-zA-Z0-9.-]+)/$', views.GridSiteSyncViewSet.as_view({'get': 'retrieve'}), - name='gridsync_singlesite' + name='gridsitesync-detail' ), re_path( r'^gridsync/(?P[a-zA-Z0-9.-]+)/(?P[0-9-]+)/$', diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index c492f88..ac47b78 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -9,6 +9,7 @@ from rest_framework import viewsets, generics from rest_framework.renderers import TemplateHTMLRenderer +from rest_framework.response import Response from monitoring.publishing.models import ( GridSite, @@ -399,12 +400,14 @@ def retrieve(self, request, SiteName=None): else: print('No need to update') - response = super(GridSiteSyncViewSet, self).list(request) - response.data = { - 'records': response.data, + sites_list_qs = GridSiteSync.objects.filter(SiteName=SiteName) + sites_list_serializer = self.get_serializer(sites_list_qs, many=True) + + response = { + 'records': sites_list_serializer.data, 'last_fetched': last_fetched } - return response + return Response(response) # Needed for passing two parameters to a viewset (GridSiteSyncSubmitHViewSet) From dcf6013a98e1167a2692eaac91d14881cf947ec8 Mon Sep 17 00:00:00 2001 From: Sae126V Date: Thu, 27 Feb 2025 01:03:07 +0000 Subject: [PATCH 105/183] Update code to make gridsync work with sitename and yearmonth - Fixed issue with HyperlinkedModelSerializer, it seems to NOT able to work with two lookup_fields before. - Simplified the split() functionality. Desc --- monitoring/publishing/serializers.py | 24 +++++++++++++++++++++--- monitoring/publishing/urls.py | 2 +- monitoring/publishing/views.py | 6 +++--- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index a4092d1..c0e6b5f 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -1,5 +1,5 @@ from rest_framework import serializers - +from rest_framework.reverse import reverse from monitoring.publishing.models import ( CloudSite, GridSite, @@ -80,6 +80,26 @@ class GridSiteSyncSubmitHSerializer(serializers.HyperlinkedModelSerializer): # ouput format. Encoding will be determined by the renderer and can be # formatted by a template filter. + # We need this because HyperlinkedModelSerializer seems to NOT able to work with two lookup_fields + class MultipleFieldLookup(serializers.HyperlinkedIdentityField): + # To match or construct the absolute URL based on the `SiteName` and `YearMonth` + def get_url(self, obj, view_name, request, format): + if not obj.SiteName or not obj.YearMonth: + return None + + return request.build_absolute_uri( + reverse( + view_name, + kwargs={ + 'SiteName': obj.SiteName, + 'YearMonth': obj.YearMonth + }, + request=request, + format=format + )) + + url = MultipleFieldLookup(view_name='gridsync-submithost') + class Meta: model = GridSiteSyncSubmitH fields = ( @@ -92,5 +112,3 @@ class Meta: 'RecordCountInDb', 'SubmitHost' ) - - lookup_fields = ('SiteName', 'YearMonth') diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py index 94c0d4b..09f7e30 100644 --- a/monitoring/publishing/urls.py +++ b/monitoring/publishing/urls.py @@ -27,7 +27,7 @@ re_path( r'^gridsync/(?P[a-zA-Z0-9.-]+)/(?P[0-9-]+)/$', views.GridSiteSyncSubmitHViewSet.as_view({'get': 'retrieve'}), - name='gridsync_submithost' + name='gridsync-submithost' ), ] diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index ac47b78..6de1a06 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -432,6 +432,7 @@ class GridSiteSyncSubmitHViewSet(MultipleFieldLookupMixin, viewsets.ReadOnlyMode queryset = GridSiteSyncSubmitH.objects.all() serializer_class = GridSiteSyncSubmitHSerializer template_name = 'gridsync_submithost.html' + lookup_fields = ('SiteName', 'YearMonth') def list(self, request): last_fetched = GridSiteSyncSubmitH.objects.aggregate(Max('fetched'))['fetched__max'] @@ -443,10 +444,8 @@ def list(self, request): return response def retrieve(self, request, SiteName=None, YearMonth=None): - - lookup_fields = ('SiteName', 'YearMonth') last_fetched = GridSiteSyncSubmitH.objects.aggregate(Max('fetched'))['fetched__max'] - Year, Month = YearMonth.replace('-', ' ').split(' ') + Year, Month = YearMonth.split('-') sitename_in_table = None yearmonth_in_table = None @@ -516,6 +515,7 @@ def retrieve(self, request, SiteName=None, YearMonth=None): right_on=['Site', 'Month', 'Year', 'SubmitHostSync'], how='outer' ) + fetchset = df_all.to_dict('index') # This is to list only data for one month From cb63154f0716567f4cb61de558b1b00ee38ac6a3 Mon Sep 17 00:00:00 2001 From: Sae126V Date: Thu, 27 Feb 2025 10:00:12 +0000 Subject: [PATCH 106/183] Fix codeclimate issues --- monitoring/publishing/serializers.py | 6 ++++-- monitoring/publishing/views.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index c0e6b5f..71b4cd7 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -25,9 +25,10 @@ class Meta: # Sitename substitutes pk lookup_field = 'SiteName' extra_kwargs = { - 'url': {'view_name': 'gridsite-detail','lookup_field': 'SiteName'} + 'url': {'view_name': 'gridsite-detail', 'lookup_field': 'SiteName'} } + class GridSiteSyncSerializer(serializers.HyperlinkedModelSerializer): # Override default format with None so that Python datetime is used as # ouput format. Encoding will be determined by the renderer and can be @@ -75,6 +76,7 @@ class Meta: 'url': {'view_name': 'cloudsite-detail', 'lookup_field': 'SiteName'} } + class GridSiteSyncSubmitHSerializer(serializers.HyperlinkedModelSerializer): # Override default format with None so that Python datetime is used as # ouput format. Encoding will be determined by the renderer and can be @@ -92,7 +94,7 @@ def get_url(self, obj, view_name, request, format): view_name, kwargs={ 'SiteName': obj.SiteName, - 'YearMonth': obj.YearMonth + 'YearMonth': obj.YearMonth }, request=request, format=format diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 6de1a06..61d639c 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -402,7 +402,7 @@ def retrieve(self, request, SiteName=None): sites_list_qs = GridSiteSync.objects.filter(SiteName=SiteName) sites_list_serializer = self.get_serializer(sites_list_qs, many=True) - + response = { 'records': sites_list_serializer.data, 'last_fetched': last_fetched From 62edf14fb403ca3b4ebbe3b2103c8e920e25fadb Mon Sep 17 00:00:00 2001 From: Sae126V Date: Thu, 27 Feb 2025 17:04:19 +0000 Subject: [PATCH 107/183] Move MultipleFieldLookup class outside --- monitoring/publishing/serializers.py | 42 ++++++++++++++++------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 71b4cd7..74b70bd 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -1,5 +1,6 @@ from rest_framework import serializers from rest_framework.reverse import reverse + from monitoring.publishing.models import ( CloudSite, GridSite, @@ -77,29 +78,34 @@ class Meta: } +class MultipleFieldLookup(serializers.HyperlinkedIdentityField): + # HyperlinkedModelSerializer seems to NOT able to work with two lookup_fields + # This class is ONLY capable to match object instance to its URL representation. + # i.e, `SiteName` and `YearMonth` ONLY + # + # Overriding the get_url() method - To match object instance to its URL representation. + def get_url(self, obj, view_name, request, format): + if not obj.SiteName or not obj.YearMonth: + return None + + return request.build_absolute_uri( + reverse( + view_name, + kwargs={ + 'SiteName': obj.SiteName, + 'YearMonth': obj.YearMonth + }, + request=request, + format=format + )) + + class GridSiteSyncSubmitHSerializer(serializers.HyperlinkedModelSerializer): # Override default format with None so that Python datetime is used as # ouput format. Encoding will be determined by the renderer and can be # formatted by a template filter. - # We need this because HyperlinkedModelSerializer seems to NOT able to work with two lookup_fields - class MultipleFieldLookup(serializers.HyperlinkedIdentityField): - # To match or construct the absolute URL based on the `SiteName` and `YearMonth` - def get_url(self, obj, view_name, request, format): - if not obj.SiteName or not obj.YearMonth: - return None - - return request.build_absolute_uri( - reverse( - view_name, - kwargs={ - 'SiteName': obj.SiteName, - 'YearMonth': obj.YearMonth - }, - request=request, - format=format - )) - + # This helps us to match or construct the absolute URL based on the `SiteName` and `YearMonth` url = MultipleFieldLookup(view_name='gridsync-submithost') class Meta: From a1821cf02bfd1263aff57263d28d8e050ba26448 Mon Sep 17 00:00:00 2001 From: Sae126V Date: Wed, 19 Mar 2025 16:01:08 +0000 Subject: [PATCH 108/183] Update code to allow underscore in an regex pattern --- monitoring/publishing/urls.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py index 09f7e30..cf0f642 100644 --- a/monitoring/publishing/urls.py +++ b/monitoring/publishing/urls.py @@ -10,22 +10,22 @@ urlpatterns = [ re_path( - r'^cloud/(?P[a-zA-Z0-9.-]+)/$', + r'^cloud/(?P[a-zA-Z0-9._-]+)/$', views.CloudSiteViewSet.as_view({'get': 'retrieve'}), name='cloudsite-detail' ), re_path( - r'^grid/(?P[a-zA-Z0-9.-]+)/$', + r'^grid/(?P[a-zA-Z0-9._-]+)/$', views.GridSiteViewSet.as_view({'get': 'retrieve'}), name='gridsite-detail' ), re_path( - r'^gridsync/(?P[a-zA-Z0-9.-]+)/$', + r'^gridsync/(?P[a-zA-Z0-9._-]+)/$', views.GridSiteSyncViewSet.as_view({'get': 'retrieve'}), name='gridsitesync-detail' ), re_path( - r'^gridsync/(?P[a-zA-Z0-9.-]+)/(?P[0-9-]+)/$', + r'^gridsync/(?P[a-zA-Z0-9._-]+)/(?P[0-9-]+)/$', views.GridSiteSyncSubmitHViewSet.as_view({'get': 'retrieve'}), name='gridsync-submithost' ), From 2d278b134d78d11148a88675d3d92cbbb081cf55 Mon Sep 17 00:00:00 2001 From: Sae126V Date: Mon, 14 Apr 2025 11:17:28 +0100 Subject: [PATCH 109/183] Update code to fix grammer for the comments --- monitoring/publishing/serializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 74b70bd..f3c42a7 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -23,7 +23,7 @@ class Meta: 'updated' ) - # Sitename substitutes pk + # Sitename substitutes for pk lookup_field = 'SiteName' extra_kwargs = { 'url': {'view_name': 'gridsite-detail', 'lookup_field': 'SiteName'} @@ -48,7 +48,7 @@ class Meta: 'SyncStatus' ) - # Sitename substitutes pk + # Sitename substitutes for pk lookup_field = 'SiteName' extra_kwargs = { 'url': {'view_name': 'gridsitesync-detail', 'lookup_field': 'SiteName'} @@ -71,7 +71,7 @@ class Meta: 'updated' ) - # Sitename substitutes pk + # Sitename substitutes for pk lookup_field = 'SiteName' extra_kwargs = { 'url': {'view_name': 'cloudsite-detail', 'lookup_field': 'SiteName'} From 3b5c92f2650a90199d82f5532c26ecc4db9a835d Mon Sep 17 00:00:00 2001 From: Sae126V Date: Tue, 29 Apr 2025 08:45:12 +0100 Subject: [PATCH 110/183] Remove lookup_field as it is been overridden --- monitoring/publishing/serializers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index f3c42a7..b273462 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -24,7 +24,6 @@ class Meta: ) # Sitename substitutes for pk - lookup_field = 'SiteName' extra_kwargs = { 'url': {'view_name': 'gridsite-detail', 'lookup_field': 'SiteName'} } @@ -49,7 +48,6 @@ class Meta: ) # Sitename substitutes for pk - lookup_field = 'SiteName' extra_kwargs = { 'url': {'view_name': 'gridsitesync-detail', 'lookup_field': 'SiteName'} } @@ -72,7 +70,6 @@ class Meta: ) # Sitename substitutes for pk - lookup_field = 'SiteName' extra_kwargs = { 'url': {'view_name': 'cloudsite-detail', 'lookup_field': 'SiteName'} } From b7fd803de0e9d9b17ef1cffabadad2fbea039c00 Mon Sep 17 00:00:00 2001 From: Sae126V Date: Wed, 30 Apr 2025 14:20:01 +0100 Subject: [PATCH 111/183] Update code to make html queryparams work --- monitoring/publishing/templates/gridsites.html | 13 +++++++------ monitoring/publishing/templates/gridsync.html | 2 +- .../publishing/templates/gridsync_singlesite.html | 2 +- monitoring/publishing/views.py | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/monitoring/publishing/templates/gridsites.html b/monitoring/publishing/templates/gridsites.html index 8abf32b..f2330a9 100644 --- a/monitoring/publishing/templates/gridsites.html +++ b/monitoring/publishing/templates/gridsites.html @@ -5,7 +5,7 @@ -APEL Publication Summary : {{ sites.0.name }} +APEL Publication Summary @@ -19,19 +19,20 @@

    APEL Publication Test

  • lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}
    - + {% for site in sites %} - - - - + + + + + {% endfor %}
    {{ sites.0.SiteName }}
    ExecutingSite MeasurementDate MeasurementTime Publication
    Status
    {{ sites.0.SiteName }}{{ last_fetched|date:"Y-m-d" }}{{ last_fetched|date:"G:i:s" }}{{ stdout }}{{ site.SiteName }}{{ site.updated|date:"Y-m-d" }}{{ site.updated|date:"G:i:s" }}{{ site.stdout }}
    diff --git a/monitoring/publishing/templates/gridsync.html b/monitoring/publishing/templates/gridsync.html index 7edccda..cf9f25b 100644 --- a/monitoring/publishing/templates/gridsync.html +++ b/monitoring/publishing/templates/gridsync.html @@ -31,7 +31,7 @@

    APEL Synchronisation Test

    {% for record in records %} - {{ record.SiteName }} + {{ record.SiteName }} {{ record.YearMonth }} {{ record.RecordStart }} {{ record.RecordEnd }} diff --git a/monitoring/publishing/templates/gridsync_singlesite.html b/monitoring/publishing/templates/gridsync_singlesite.html index f6e9dc5..169a043 100644 --- a/monitoring/publishing/templates/gridsync_singlesite.html +++ b/monitoring/publishing/templates/gridsync_singlesite.html @@ -30,7 +30,7 @@

    APEL Synchronisation Test

    {% for record in records %} - {{ record.YearMonth }} + {{ record.YearMonth }} {{ record.RecordStart }} {{ record.RecordEnd }} {{ record.RecordCountPublished }} diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 61d639c..2a44da9 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -194,6 +194,7 @@ def retrieve(self, request, SiteName=None): response = super(GridSiteViewSet, self).retrieve(request) date = response.data['updated'].replace(tzinfo=None) + response.data = update_dict_stdout_and_returncode(response.data, date) # Wrap data in a dict so that it can display in template. if type(request.accepted_renderer) is TemplateHTMLRenderer: @@ -203,7 +204,6 @@ def retrieve(self, request, SiteName=None): 'last_fetched': last_fetched } - response.data = update_dict_stdout_and_returncode(response.data, date) return response From 047d59bb9fd2c0ffd8f186ce47499b18f3383ba3 Mon Sep 17 00:00:00 2001 From: Sae126V Date: Tue, 29 Apr 2025 10:18:39 +0100 Subject: [PATCH 112/183] Initial commit to work around slow SQL queries This change adds a standalone script to update the local Django database from the APEL database on a cron. This means that the webpages remain responsive. --- monitoring/publishing/db_update_sqlite.py | 279 ++++++++++++++++++++++ monitoring/publishing/views.py | 250 +------------------ monitoring/settings.ini | 2 + 3 files changed, 283 insertions(+), 248 deletions(-) create mode 100644 monitoring/publishing/db_update_sqlite.py diff --git a/monitoring/publishing/db_update_sqlite.py b/monitoring/publishing/db_update_sqlite.py new file mode 100644 index 0000000..71b5882 --- /dev/null +++ b/monitoring/publishing/db_update_sqlite.py @@ -0,0 +1,279 @@ +# -*- coding: utf-8 -*- +""" +`db_update_sqlite.py` - Syncs data from external database into local SQLite DB. + - It will be run as a standalone operation via cron. +""" +import configparser +import logging +import os +import sys + +import pandas as pd +import django +from django.db import DatabaseError + + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Find the root and the Django project +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) + +try: + # Read configuration from the file + cp = configparser.ConfigParser(interpolation=None) + file_path = os.path.join(BASE_DIR, 'monitoring', 'settings.ini') + cp.read(file_path) + +except (configparser.NoSectionError) as err: + print("Error in configuration file. Check that file exists first: %s" % err) + sys.exit(1) + +# Set up basic logging config +logging.basicConfig( + filename=cp.get('common', 'logfile'), + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) + +# set up the logger +log = logging.getLogger(__name__) + +# Set up Django settings to run this `db_update_sqlite.py` as standalone file +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "monitoring.settings") + +# Initialize and setup Django +django.setup() + + +from monitoring.publishing.models import ( + GridSite, + CloudSite, + GridSiteSync, + VAnonCloudRecord, + VSuperSummaries, + VSyncRecords +) + +summaries_dict_standard = { + "Site": [], + "Month": [], + "Year": [], + "RecordCountPublished": [], + "RecordStart": [], + "RecordEnd": [], + "SubmitHostSumm": [], +} + +syncrecords_dict_standard = { + "Site": [], + "Month": [], + "Year": [], + "RecordCountInDb": [], + "SubmitHostSync": [] +} + + +def fill_summaries_dict(inpDict, row): + fields_to_update_and_value_to_add = { + "Site": row.Site, + "Month": row.Month, + "Year": row.Year, + "RecordCountPublished": row.RecordCountPublished, + "RecordStart": row.RecordStart, + "RecordEnd": row.RecordEnd, + } + + for field, value in fields_to_update_and_value_to_add.items(): + inpDict[field] = inpDict.get(field) + [value] + + if hasattr(row, "SubmitHostSumm"): + inpDict["SubmitHostSumm"] = inpDict.get("SubmitHostSumm") + [row.SubmitHostSumm] + + return inpDict + +def fill_syncrecords_dict(inpDict, row): + inpDict["Site"] = inpDict.get("Site") + [row.Site] + inpDict["Month"] = inpDict.get("Month") + [row.Month] + inpDict["Year"] = inpDict.get("Year") + [row.Year] + inpDict["RecordCountInDb"] = inpDict.get("RecordCountInDb") + [row.RecordCountInDb] + if hasattr(row, "SubmitHostSync"): + inpDict["SubmitHostSync"] = inpDict.get("SubmitHostSync") + [row.SubmitHostSync] + return inpDict + +def correct_dict(inpDict): + keys_to_remove = [] + for key, val in inpDict.items(): + if len(val) == 0: + keys_to_remove.append(key) + for key in keys_to_remove: + inpDict.pop(key) + return inpDict + +def get_year_month_str(year, month): + year_string = str(year) + month_string = str(month) + if len(month_string) == 1: + month_string = '0' + month_string + return year_string + '-' + month_string + +def determine_sync_status(f): + RecordCountPublished = f.get("RecordCountPublished") + RecordCountInDb = f.get("RecordCountInDb") + rel_diff1 = abs(RecordCountPublished - RecordCountInDb) / RecordCountInDb + rel_diff2 = abs(RecordCountPublished - RecordCountInDb) / RecordCountPublished + if rel_diff1 < 0.01 or rel_diff2 < 0.01: + return "OK" + return "ERROR [ Please use the Gap Publisher to synchronise this dataset]" + + +def refresh_gridsite(): + try: + sql_query = """ + SELECT + Site, + max(LatestEndTime) AS LatestPublish + FROM VSuperSummaries + GROUP BY 1; + """ + fetchset = VSuperSummaries.objects.using('grid').raw(sql_query) + + for f in fetchset: + GridSite.objects.update_or_create( + defaults={'updated': f.LatestPublish}, + SiteName=f.Site + ) + + log.info("Refreshed GridSite") + + except DatabaseError: + log.exception('Error while trying to refresh GridSite') + + +def refresh_cloudsite(): + try: + sql_query = """ + SELECT + b.SiteName, + COUNT(DISTINCT VMUUID) as VMs, + CloudType, + b.UpdateTime + FROM( + SELECT + SiteName, + MAX(UpdateTime) AS latest + FROM VAnonCloudRecords + WHERE UpdateTime>'2023-01-01' + GROUP BY SiteName + ) + AS a + INNER JOIN VAnonCloudRecords + AS b + ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest + GROUP BY SiteName; + """ + fetchset = VAnonCloudRecord.objects.using('cloud').raw(sql_query) + + for f in fetchset: + CloudSite.objects.update_or_create( + defaults={ + 'Vms': f.VMs, + 'Script': f.CloudType, + 'updated': f.UpdateTime + }, + SiteName=f.SiteName + ) + + log.info("Refreshed CloudSite") + + except DatabaseError: + log.exception('Error while trying to refresh CloudSite') + + +def refresh_gridsitesync(): + try: + # The condition on EarliestEndTime and LatestEndTime is necessary to avoid error by pytz because of dates like '00-00-00' + sql_query_summaries = """ + SELECT + Site, + Month, + Year, + SUM(NumberOfJobs) AS RecordCountPublished, + MIN(EarliestEndTime) AS RecordStart, + MAX(LatestEndTime) AS RecordEnd + FROM VSuperSummaries + WHERE + EarliestEndTime>'1900-01-01' AND + LatestEndTime>'1900-01-01' + GROUP BY + Site, Year, Month; + """ + fetchset_Summaries = VSuperSummaries.objects.using('grid').raw(sql_query_summaries) + + sql_query_syncrec = """ + SELECT + Site, + Month, + Year, + SUM(NumberOfJobs) AS RecordCountInDb + FROM VSyncRecords + GROUP BY + Site, Year, Month; + """ + fetchset_SyncRecords = VSyncRecords.objects.using('grid').raw(sql_query_syncrec) + + # Create empty dicts that will become dfs to be combined + summaries_dict = summaries_dict_standard.copy() + syncrecords_dict = syncrecords_dict_standard.copy() + + # Fill the dicts with the fetched data + for row in fetchset_Summaries: + summaries_dict = fill_summaries_dict(summaries_dict, row) + summaries_dict = correct_dict(summaries_dict) + for row in fetchset_SyncRecords: + syncrecords_dict = fill_syncrecords_dict(syncrecords_dict, row) + syncrecords_dict = correct_dict(syncrecords_dict) + + # Merge data from VSuperSummaries and VSyncRecords into one df + df_Summaries = pd.DataFrame.from_dict(summaries_dict) + df_SyncRecords = pd.DataFrame.from_dict(syncrecords_dict) + df_all = df_Summaries.merge( + df_SyncRecords, + left_on=['Site', 'Month', 'Year'], + right_on=['Site', 'Month', 'Year'], + how='inner' + ) + fetchset = df_all.to_dict('index') + + # Determine SyncStatus based on the difference between records published and in db + for f in fetchset.values(): + f['SyncStatus'] = determine_sync_status(f) + + # Combined primary keys outside the default dict + GridSiteSync.objects.update_or_create( + defaults={ + 'RecordStart': f.get("RecordStart"), + 'RecordEnd': f.get("RecordEnd"), + 'RecordCountPublished': f.get("RecordCountPublished"), + 'RecordCountInDb': f.get("RecordCountInDb"), + 'SyncStatus': f.get("SyncStatus"), + }, + YearMonth=get_year_month_str(f.get("Year"), f.get("Month")), + SiteName=f.get("Site"), + Month=f.get("Month"), + Year=f.get("Year"), + ) + log.info("Refreshed GridSiteSync") + + except DatabaseError: + log.exception('Error while trying to refresh GridSiteSync') + +if __name__ == "__main__": + + refresh_gridsite() + refresh_cloudsite() + refresh_gridsitesync() + + log.info( + "Data retrieval from the database backend(s) is completed and " + "successfully synchronized with the local SQLite database." + ) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 2a44da9..37fc014 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -7,15 +7,15 @@ from django.shortcuts import get_object_or_404 import pandas as pd -from rest_framework import viewsets, generics +from rest_framework import viewsets from rest_framework.renderers import TemplateHTMLRenderer from rest_framework.response import Response + from monitoring.publishing.models import ( GridSite, VSuperSummaries, CloudSite, - VAnonCloudRecord, GridSiteSync, VSyncRecords, GridSiteSyncSubmitH @@ -134,23 +134,6 @@ def list(self, request): last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max'] if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - if last_fetched is None or (last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20))): - sql_query = """ - SELECT - Site, - max(LatestEndTime) AS LatestPublish - FROM VSuperSummaries - GROUP BY 1; - """ - fetchset = VSuperSummaries.objects.using('grid').raw(sql_query) - - for f in fetchset: - GridSite.objects.update_or_create( - defaults={'updated': f.LatestPublish}, - SiteName=f.Site - ) - else: - print('No need to update') final_response = [] response = super(GridSiteViewSet, self).list(request) @@ -173,24 +156,6 @@ def retrieve(self, request, SiteName=None): # If there's no data then last_fetched is None. if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): - print('Out of date') - sql_query = """ - SELECT - Site, - max(LatestEndTime) AS LatestPublish - FROM VSuperSummaries - GROUP BY 1; - """ - fetchset = VSuperSummaries.objects.using('grid').raw(sql_query) - - for f in fetchset: - GridSite.objects.update_or_create( - defaults={'updated': f.LatestPublish}, - SiteName=f.Site - ) - else: - print('No need to update') response = super(GridSiteViewSet, self).retrieve(request) date = response.data['updated'].replace(tzinfo=None) @@ -222,89 +187,9 @@ def get_template_names(self): def list(self, request): last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] - n_sites = GridSiteSync.objects.values('SiteName').distinct().count() if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or n_sites == 1: - print('Out of date') - - # The condition on EarliestEndTime and LatestEndTime is necessary to avoid error by pytz because of dates like '00-00-00' - sql_query_summaries = """ - SELECT - Site, - Month, Year, - SUM(NumberOfJobs) AS RecordCountPublished, - MIN(EarliestEndTime) AS RecordStart, - MAX(LatestEndTime) AS RecordEnd - FROM VSuperSummaries - WHERE - EarliestEndTime>'1900-01-01' AND - LatestEndTime>'1900-01-01' - GROUP BY - Site, Year, Month; - """ - fetchset_Summaries = VSuperSummaries.objects.using('grid').raw(sql_query_summaries) - - sql_query_syncrec = """ - SELECT - Site, - Month, - Year, - SUM(NumberOfJobs) AS RecordCountInDb - FROM VSyncRecords - GROUP BY - Site, Year, Month; - """ - fetchset_SyncRecords = VSyncRecords.objects.using('grid').raw(sql_query_syncrec) - - # Create empty dicts that will become dfs to be combined - summaries_dict = summaries_dict_standard.copy() - syncrecords_dict = syncrecords_dict_standard.copy() - - # Fill the dicts with the fetched data - for row in fetchset_Summaries: - summaries_dict = fill_summaries_dict(summaries_dict, row) - summaries_dict = correct_dict(summaries_dict) - for row in fetchset_SyncRecords: - syncrecords_dict = fill_syncrecords_dict(syncrecords_dict, row) - syncrecords_dict = correct_dict(syncrecords_dict) - - # Merge data from VSuperSummaries and VSyncRecords into one df - df_Summaries = pd.DataFrame.from_dict(summaries_dict) - df_SyncRecords = pd.DataFrame.from_dict(syncrecords_dict) - df_all = df_Summaries.merge( - df_SyncRecords, - left_on=['Site', 'Month', 'Year'], - right_on=['Site', 'Month', 'Year'], - how='inner' - ) - fetchset = df_all.to_dict('index') - - # Delete all data if table not empty (as this function lists all sites) - GridSiteSync.objects.all().delete() - - # Determine SyncStatus based on the difference between records published and in db - for f in fetchset.values(): - f['SyncStatus'] = determine_sync_status(f) - - # Combined primary keys outside the default dict - GridSiteSync.objects.update_or_create( - defaults={ - 'RecordStart': f.get("RecordStart"), - 'RecordEnd': f.get("RecordEnd"), - 'RecordCountPublished': f.get("RecordCountPublished"), - 'RecordCountInDb': f.get("RecordCountInDb"), - 'SyncStatus': f.get("SyncStatus"), - }, - YearMonth=get_year_month_str(f.get("Year"), f.get("Month")), - SiteName=f.get("Site"), - Month=f.get("Month"), - Year=f.get("Year"), - ) - - else: - print('No need to update') response = super(GridSiteSyncViewSet, self).list(request) response.data = { @@ -315,90 +200,9 @@ def list(self, request): def retrieve(self, request, SiteName=None): last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] - row_1 = GridSiteSync.objects.filter()[:1].get() - n_sites = GridSiteSync.objects.values('SiteName').distinct().count() if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or n_sites > 1 or SiteName != row_1.SiteName: - print('Out of date') - - # The condition on EarliestEndTime and LatestEndTime is necessary to avoid error by pytz because of dates like '00-00-00' - sql_query_summaries = """ - SELECT - Site, - Month, - Year, - SUM(NumberOfJobs) AS RecordCountPublished, - MIN(EarliestEndTime) AS RecordStart, - MAX(LatestEndTime) AS RecordEnd - FROM VSuperSummaries - WHERE - Site='{}' AND - EarliestEndTime>'1900-01-01' AND - LatestEndTime>'1900-01-01' - GROUP BY - Site, Year, Month; - """.format(SiteName) - fetchset_Summaries = VSuperSummaries.objects.using('grid').raw(sql_query_summaries) - - sql_query_syncrecords = """ - SELECT - Site, - Month, - Year, - SUM(NumberOfJobs) AS RecordCountInDb - FROM VSyncRecords - WHERE Site='{}' - GROUP BY - Site, Year, Month; - """.format(SiteName) - fetchset_SyncRecords = VSyncRecords.objects.using('grid').raw(sql_query_syncrecords) - - summaries_dict = summaries_dict_standard.copy() - syncrecords_dict = syncrecords_dict_standard.copy() - - for row in fetchset_Summaries: - summaries_dict = fill_summaries_dict(summaries_dict, row) - summaries_dict = correct_dict(summaries_dict) - for row in fetchset_SyncRecords: - syncrecords_dict = fill_syncrecords_dict(syncrecords_dict, row) - syncrecords_dict = correct_dict(syncrecords_dict) - - df_Summaries = pd.DataFrame.from_dict(summaries_dict) - df_SyncRecords = pd.DataFrame.from_dict(syncrecords_dict) - df_all = df_Summaries.merge( - df_SyncRecords, - left_on=['Site', 'Month', 'Year'], - right_on=['Site', 'Month', 'Year'], - how='inner' - ) - fetchset = df_all.to_dict('index') - - # Ensure we list only the data for one site - first_row = GridSiteSync.objects.first() - if hasattr(first_row, "SiteName") and first_row.SiteName != SiteName: - GridSiteSync.objects.all().delete() - - for f in fetchset.values(): - f['SyncStatus'] = determine_sync_status(f) - - GridSiteSync.objects.update_or_create( - defaults={ - 'RecordStart': f.get("RecordStart"), - 'RecordEnd': f.get("RecordEnd"), - 'RecordCountPublished': f.get("RecordCountPublished"), - 'RecordCountInDb': f.get("RecordCountInDb"), - 'SyncStatus': f.get("SyncStatus"), - }, - YearMonth=get_year_month_str(f.get("Year"), f.get("Month")), - SiteName=f.get("Site"), - Month=f.get("Month"), - Year=f.get("Year"), - ) - - else: - print('No need to update') sites_list_qs = GridSiteSync.objects.filter(SiteName=SiteName) sites_list_serializer = self.get_serializer(sites_list_qs, many=True) @@ -557,42 +361,6 @@ def list(self, request): last_fetched = CloudSite.objects.aggregate(Max('fetched'))['fetched__max'] if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - if last_fetched is None or (last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20))): - print('Out of date') - - sql_query = """ - SELECT - b.SiteName, - COUNT(DISTINCT VMUUID) as VMs, - CloudType, - b.UpdateTime - FROM( - SELECT - SiteName, - MAX(UpdateTime) AS latest - FROM VAnonCloudRecords - WHERE UpdateTime>'2023-01-01' - GROUP BY SiteName - ) - AS a - INNER JOIN VAnonCloudRecords - AS b - ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest - GROUP BY SiteName; - """ - fetchset = VAnonCloudRecord.objects.using('cloud').raw(sql_query) - - for f in fetchset: - CloudSite.objects.update_or_create( - defaults={ - 'Vms': f.VMs, - 'Script': f.CloudType, - 'updated': f.UpdateTime - }, - SiteName=f.SiteName - ) - else: - print('No need to update') response = super(CloudSiteViewSet, self).list(request) # Wrap data in a dict so that it can display in template. @@ -606,20 +374,6 @@ def list(self, request): def retrieve(self, request, SiteName=None): last_fetched = CloudSite.objects.aggregate(Max('fetched'))['fetched__max'] print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - if last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): - print('Out of date') - fetchset = VAnonCloudRecord.objects.using('cloud').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID) as VMs, CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") - for f in fetchset: - CloudSite.objects.update_or_create( - defaults={ - 'Vms': f.VMs, - 'Script': f.CloudType, - 'updated': f.UpdateTime - }, - SiteName=f.SiteName - ) - else: - print('No need to update') response = super(CloudSiteViewSet, self).retrieve(request) # Wrap data in a dict so that it can display in template. diff --git a/monitoring/settings.ini b/monitoring/settings.ini index 222b25c..fdb8b35 100644 --- a/monitoring/settings.ini +++ b/monitoring/settings.ini @@ -7,6 +7,8 @@ secret_key = # `allowed_hosts` values should be comma separated list of hostnames (fqdn's) allowed_hosts = +# Path to the log file for `db_update_sqlite.py` execution info. +logfile = # Information about the database connection - grid [db_grid] From 3d0cec1b90da34b39b8e9423329fe68f73b2cdef Mon Sep 17 00:00:00 2001 From: Sae126V Date: Fri, 2 May 2025 09:18:44 +0100 Subject: [PATCH 113/183] Add pagination to gridsync records This page returns a large number of results so needs pagination to not grind to a halt. --- monitoring/publishing/db_update_sqlite.py | 4 ++++ monitoring/publishing/templates/gridsync.html | 2 +- monitoring/publishing/views.py | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/monitoring/publishing/db_update_sqlite.py b/monitoring/publishing/db_update_sqlite.py index 71b5882..0d415aa 100644 --- a/monitoring/publishing/db_update_sqlite.py +++ b/monitoring/publishing/db_update_sqlite.py @@ -91,6 +91,7 @@ def fill_summaries_dict(inpDict, row): return inpDict + def fill_syncrecords_dict(inpDict, row): inpDict["Site"] = inpDict.get("Site") + [row.Site] inpDict["Month"] = inpDict.get("Month") + [row.Month] @@ -100,6 +101,7 @@ def fill_syncrecords_dict(inpDict, row): inpDict["SubmitHostSync"] = inpDict.get("SubmitHostSync") + [row.SubmitHostSync] return inpDict + def correct_dict(inpDict): keys_to_remove = [] for key, val in inpDict.items(): @@ -109,6 +111,7 @@ def correct_dict(inpDict): inpDict.pop(key) return inpDict + def get_year_month_str(year, month): year_string = str(year) month_string = str(month) @@ -267,6 +270,7 @@ def refresh_gridsitesync(): except DatabaseError: log.exception('Error while trying to refresh GridSiteSync') + if __name__ == "__main__": refresh_gridsite() diff --git a/monitoring/publishing/templates/gridsync.html b/monitoring/publishing/templates/gridsync.html index cf9f25b..c663b1c 100644 --- a/monitoring/publishing/templates/gridsync.html +++ b/monitoring/publishing/templates/gridsync.html @@ -29,7 +29,7 @@

    APEL Synchronisation Test

    Synchronisation
    Status - {% for record in records %} + {% for record in records.results %} {{ record.SiteName }} {{ record.YearMonth }} diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 37fc014..8732b21 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -10,6 +10,7 @@ from rest_framework import viewsets from rest_framework.renderers import TemplateHTMLRenderer from rest_framework.response import Response +from rest_framework.pagination import PageNumberPagination from monitoring.publishing.models import ( @@ -172,10 +173,15 @@ def retrieve(self, request, SiteName=None): return response +class GridSiteSyncPagination(PageNumberPagination): + page_size = 1000 # Number of items to be fetched per page + + class GridSiteSyncViewSet(viewsets.ReadOnlyModelViewSet): queryset = GridSiteSync.objects.all() serializer_class = GridSiteSyncSerializer lookup_field = 'SiteName' + pagination_class = GridSiteSyncPagination # When a single site is showed (retrieve function used), the template # is different than the one used when showing a list of sites From 69f4b74ff4c2db3e7b1508a8382e140b0c2d0273 Mon Sep 17 00:00:00 2001 From: Sae126V Date: Thu, 8 May 2025 18:20:40 +0100 Subject: [PATCH 114/183] Improve styling and comments, and reuse functions --- monitoring/publishing/db_update_sqlite.py | 131 ++++++---------------- monitoring/publishing/views.py | 2 +- 2 files changed, 36 insertions(+), 97 deletions(-) diff --git a/monitoring/publishing/db_update_sqlite.py b/monitoring/publishing/db_update_sqlite.py index 0d415aa..1f65076 100644 --- a/monitoring/publishing/db_update_sqlite.py +++ b/monitoring/publishing/db_update_sqlite.py @@ -8,15 +8,44 @@ import os import sys -import pandas as pd import django from django.db import DatabaseError +import pandas as pd -BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) # Find the root and the Django project -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +sys.path.append(BASE_DIR) + +# Set up Django settings to run this `db_update_sqlite.py` as standalone file +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "monitoring.settings") + +# Initialize and setup Django +django.setup() + + +# Cron jobs run in a minimal environment and lack access to Django settings. +# To ensure proper model imports and database interactions, we MUST initialize and setup Django first. +from monitoring.publishing.models import ( + GridSite, + CloudSite, + GridSiteSync, + VAnonCloudRecord, + VSuperSummaries, + VSyncRecords +) + +from monitoring.publishing.views import ( + summaries_dict_standard, + syncrecords_dict_standard, + correct_dict, + determine_sync_status, + fill_summaries_dict, + fill_syncrecords_dict, + get_year_month_str +) + try: # Read configuration from the file @@ -38,96 +67,6 @@ # set up the logger log = logging.getLogger(__name__) -# Set up Django settings to run this `db_update_sqlite.py` as standalone file -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "monitoring.settings") - -# Initialize and setup Django -django.setup() - - -from monitoring.publishing.models import ( - GridSite, - CloudSite, - GridSiteSync, - VAnonCloudRecord, - VSuperSummaries, - VSyncRecords -) - -summaries_dict_standard = { - "Site": [], - "Month": [], - "Year": [], - "RecordCountPublished": [], - "RecordStart": [], - "RecordEnd": [], - "SubmitHostSumm": [], -} - -syncrecords_dict_standard = { - "Site": [], - "Month": [], - "Year": [], - "RecordCountInDb": [], - "SubmitHostSync": [] -} - - -def fill_summaries_dict(inpDict, row): - fields_to_update_and_value_to_add = { - "Site": row.Site, - "Month": row.Month, - "Year": row.Year, - "RecordCountPublished": row.RecordCountPublished, - "RecordStart": row.RecordStart, - "RecordEnd": row.RecordEnd, - } - - for field, value in fields_to_update_and_value_to_add.items(): - inpDict[field] = inpDict.get(field) + [value] - - if hasattr(row, "SubmitHostSumm"): - inpDict["SubmitHostSumm"] = inpDict.get("SubmitHostSumm") + [row.SubmitHostSumm] - - return inpDict - - -def fill_syncrecords_dict(inpDict, row): - inpDict["Site"] = inpDict.get("Site") + [row.Site] - inpDict["Month"] = inpDict.get("Month") + [row.Month] - inpDict["Year"] = inpDict.get("Year") + [row.Year] - inpDict["RecordCountInDb"] = inpDict.get("RecordCountInDb") + [row.RecordCountInDb] - if hasattr(row, "SubmitHostSync"): - inpDict["SubmitHostSync"] = inpDict.get("SubmitHostSync") + [row.SubmitHostSync] - return inpDict - - -def correct_dict(inpDict): - keys_to_remove = [] - for key, val in inpDict.items(): - if len(val) == 0: - keys_to_remove.append(key) - for key in keys_to_remove: - inpDict.pop(key) - return inpDict - - -def get_year_month_str(year, month): - year_string = str(year) - month_string = str(month) - if len(month_string) == 1: - month_string = '0' + month_string - return year_string + '-' + month_string - -def determine_sync_status(f): - RecordCountPublished = f.get("RecordCountPublished") - RecordCountInDb = f.get("RecordCountInDb") - rel_diff1 = abs(RecordCountPublished - RecordCountInDb) / RecordCountInDb - rel_diff2 = abs(RecordCountPublished - RecordCountInDb) / RecordCountPublished - if rel_diff1 < 0.01 or rel_diff2 < 0.01: - return "OK" - return "ERROR [ Please use the Gap Publisher to synchronise this dataset]" - def refresh_gridsite(): try: @@ -174,7 +113,7 @@ def refresh_cloudsite(): ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName; """ - fetchset = VAnonCloudRecord.objects.using('cloud').raw(sql_query) + fetchset = VAnonCloudRecord.objects.using('cloud').raw(sql_query) for f in fetchset: CloudSite.objects.update_or_create( @@ -278,6 +217,6 @@ def refresh_gridsitesync(): refresh_gridsitesync() log.info( - "Data retrieval from the database backend(s) is completed and " - "successfully synchronized with the local SQLite database." + "Data retrieval and processing attempted. " + "Check the above logs for details on the sync status" ) diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 8732b21..6755a02 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -174,7 +174,7 @@ def retrieve(self, request, SiteName=None): class GridSiteSyncPagination(PageNumberPagination): - page_size = 1000 # Number of items to be fetched per page + page_size = 1000 # Number of items to be fetched per page class GridSiteSyncViewSet(viewsets.ReadOnlyModelViewSet): From 1a81643e3a034ed0b15437a0e6330f8e4a45f3ae Mon Sep 17 00:00:00 2001 From: Sae126V Date: Fri, 16 May 2025 17:39:04 +0100 Subject: [PATCH 115/183] Add log separators and move function to script The function is no-longer used in views, so move it to the script it's used in. --- monitoring/publishing/db_update_sqlite.py | 18 +++++++++++++++++- monitoring/publishing/views.py | 12 ------------ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/monitoring/publishing/db_update_sqlite.py b/monitoring/publishing/db_update_sqlite.py index 1f65076..2bd7504 100644 --- a/monitoring/publishing/db_update_sqlite.py +++ b/monitoring/publishing/db_update_sqlite.py @@ -40,7 +40,6 @@ summaries_dict_standard, syncrecords_dict_standard, correct_dict, - determine_sync_status, fill_summaries_dict, fill_syncrecords_dict, get_year_month_str @@ -68,6 +67,21 @@ log = logging.getLogger(__name__) +def determine_sync_status(f): + """ + Helper to determine sync status between published and the database record counts. + """ + RecordCountPublished = f.get("RecordCountPublished") + RecordCountInDb = f.get("RecordCountInDb") + rel_diff1 = abs(RecordCountPublished - RecordCountInDb)/RecordCountInDb + rel_diff2 = abs(RecordCountPublished - RecordCountInDb)/RecordCountPublished + if rel_diff1 < 0.01 or rel_diff2 < 0.01: + syncstatus = "OK" + else: + syncstatus = "ERROR [ Please use the Gap Publisher to synchronise this dataset]" + return syncstatus + + def refresh_gridsite(): try: sql_query = """ @@ -211,6 +225,7 @@ def refresh_gridsitesync(): if __name__ == "__main__": + log.info('=====================') refresh_gridsite() refresh_cloudsite() @@ -220,3 +235,4 @@ def refresh_gridsitesync(): "Data retrieval and processing attempted. " "Check the above logs for details on the sync status" ) + log.info('=====================') diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 6755a02..0e07e74 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -104,18 +104,6 @@ def correct_dict(inpDict): return inpDict -def determine_sync_status(f): - RecordCountPublished = f.get("RecordCountPublished") - RecordCountInDb = f.get("RecordCountInDb") - rel_diff1 = abs(RecordCountPublished - RecordCountInDb)/RecordCountInDb - rel_diff2 = abs(RecordCountPublished - RecordCountInDb)/RecordCountPublished - if rel_diff1 < 0.01 or rel_diff2 < 0.01: - syncstatus = "OK" - else: - syncstatus = "ERROR [ Please use the Gap Publisher to synchronise this dataset]" - return syncstatus - - # Combine Year and Month into one string (display purposes) def get_year_month_str(year, month): year_string = str(year) From ed2c9c2e5daf910793c7c0cecffc71fae175738a Mon Sep 17 00:00:00 2001 From: Sae126V Date: Fri, 13 Jun 2025 12:41:53 +0100 Subject: [PATCH 116/183] Set HTML as a default renderer and remove admin renderer --- monitoring/settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/monitoring/settings.py b/monitoring/settings.py index f427268..627800e 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -80,10 +80,9 @@ REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ( + 'rest_framework.renderers.TemplateHTMLRenderer', 'rest_framework.renderers.JSONRenderer', 'rest_framework.renderers.BrowsableAPIRenderer', - 'rest_framework.renderers.TemplateHTMLRenderer', - 'rest_framework.renderers.AdminRenderer', ) } From 02062b53e87db4351a70529e2c627b9313beb583 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 17 Jun 2025 17:20:51 +0100 Subject: [PATCH 117/183] Update version number to 0.2 This should have been done before tagging the v0.2 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 168bf46..307434f 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name='monitoring', - version='0.1', + version='0.2', packages=find_packages(), scripts=['manage.py'], ) From 946ee5a079052d7cb1c77e288425af7f02b507e3 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 17 Jun 2025 17:08:41 +0100 Subject: [PATCH 118/183] Update reqs to last versions that support Py3.6 This is a stepping stone to updating the host OS and then the Python version. --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 581d42c..41b5b0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # Pin packages to support and work with py3.6. -Django==3.1.14 +Django==3.2.25 djangorestframework==3.15.1 -pytz==2024.2 +pytz==2025.2 PyMySQL==1.0.2 pandas==1.1.5 From 323ee1e789f3ac76e84e487a85aded8ebf5547ca Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 17 Jun 2025 17:21:08 +0100 Subject: [PATCH 119/183] Set app names to dotted paths relative to base dir Later versions of Django are particular about requiring the app name in the app's Config to be a full path from the root (where manage.py is). --- monitoring/availability/apps.py | 2 +- monitoring/publishing/apps.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/availability/apps.py b/monitoring/availability/apps.py index c7eacd8..53f90fa 100644 --- a/monitoring/availability/apps.py +++ b/monitoring/availability/apps.py @@ -5,4 +5,4 @@ class AvailabilityConfig(AppConfig): - name = 'availability' + name = 'monitoring.availability' diff --git a/monitoring/publishing/apps.py b/monitoring/publishing/apps.py index 57c17f7..d5b06c3 100644 --- a/monitoring/publishing/apps.py +++ b/monitoring/publishing/apps.py @@ -5,4 +5,4 @@ class PublishingConfig(AppConfig): - name = 'publishing' + name = 'monitoring.publishing' From 5b23760892f9c44c3d9e1dad108180f074e4b602 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 18 Jun 2025 14:42:09 +0100 Subject: [PATCH 120/183] Update version numbers and formatting in docs --- docs/what_gets_installed.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/what_gets_installed.md b/docs/what_gets_installed.md index ade3ecd..38d7d38 100644 --- a/docs/what_gets_installed.md +++ b/docs/what_gets_installed.md @@ -2,20 +2,20 @@ For Django to work with apache, it is common to have a venv within the app, wher ## Packages installed by Aquilon outside the venv Following the config file that Aquilon uses, the following are the packages installed: -- httpd -- python3-mod_wsgi (for apache to work with django) -- python3-devel -- gcc (needed for dependencies) -- mariadb -- tar. +- `httpd` +- `python3-mod_wsgi` (for apache to work with django) +- `python3-devel` +- `gcc` (needed for dependencies) +- `mariadb` +- `tar` ## Packages installed within the venv Within venv, the following are installed through pip: -- djangorestframework (3.15.1) -- pymysql (1.0.2) (needed for mariadb to work) -- pandas (1.1.5) (needed by the app) -- django (3.1.14) -- pytz (2024.2). +- `djangorestframework` (3.15.1) +- `pymysql` (1.0.2) (needed for mariadb to work) +- `pandas` (1.1.5) (needed by the app) +- `django` (3.2.25) +- `pytz` (2025.2) Note that when the version of the packages is specified, the app would not work with a different version (due to dependencies conflicts). From 0170e950141d4fc69130e061548f5fc1de4acecc Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 19 Jun 2025 11:08:12 +0100 Subject: [PATCH 121/183] Update version number for v0.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 307434f..030d2c4 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name='monitoring', - version='0.2', + version='0.3', packages=find_packages(), scripts=['manage.py'], ) From f5d55907ddadbaf96ace1df318c83b842e18cf3c Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 19 Jun 2025 12:47:47 +0100 Subject: [PATCH 122/183] Filter all db update queries to last 3 years We don't really care about data past this point, so fetching less should improve performance a bit and not overwhelm the interface. The description on the cloud page is updated to match. --- monitoring/publishing/db_update_sqlite.py | 6 +++++- monitoring/publishing/templates/cloudsites.html | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/db_update_sqlite.py b/monitoring/publishing/db_update_sqlite.py index 2bd7504..81a597f 100644 --- a/monitoring/publishing/db_update_sqlite.py +++ b/monitoring/publishing/db_update_sqlite.py @@ -89,6 +89,7 @@ def refresh_gridsite(): Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries + WHERE LatestEndTime > DATE_SUB(NOW(), INTERVAL 3 YEAR) GROUP BY 1; """ fetchset = VSuperSummaries.objects.using('grid').raw(sql_query) @@ -118,7 +119,7 @@ def refresh_cloudsite(): SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords - WHERE UpdateTime>'2023-01-01' + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 YEAR) GROUP BY SiteName ) AS a @@ -158,6 +159,7 @@ def refresh_gridsitesync(): MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE + Year >= YEAR(NOW()) - 3 AND EarliestEndTime>'1900-01-01' AND LatestEndTime>'1900-01-01' GROUP BY @@ -172,6 +174,8 @@ def refresh_gridsitesync(): Year, SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords + WHERE + Year >= YEAR(NOW()) - 3 GROUP BY Site, Year, Month; """ diff --git a/monitoring/publishing/templates/cloudsites.html b/monitoring/publishing/templates/cloudsites.html index 5fa8680..871ad9d 100644 --- a/monitoring/publishing/templates/cloudsites.html +++ b/monitoring/publishing/templates/cloudsites.html @@ -8,7 +8,7 @@ -

    Sites publishing cloud accounting records from 2018-06-19 onwards

    +

    Sites publishing cloud accounting records in last 3 years

    Page last updated: {{ last_fetched|date:"Y-m-d H:i:s.u" }}

    From 3005e4cf4a277c6521924026117a4ddfe5c3a8ae Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 19 Jun 2025 16:28:51 +0100 Subject: [PATCH 123/183] Update version to v0.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 030d2c4..f21ce6e 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name='monitoring', - version='0.3', + version='0.4', packages=find_packages(), scripts=['manage.py'], ) From f51396eef73c4f2e4c5c6585756c57ee4da2f08c Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 23 Jun 2025 17:22:10 +0100 Subject: [PATCH 124/183] Pin numpy version for Py3.6 compatibility --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 41b5b0c..69f5922 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ djangorestframework==3.15.1 pytz==2025.2 PyMySQL==1.0.2 pandas==1.1.5 +numpy==1.19.5 # pandas dependency From 7583b39a53d835ba6bd759daa89c48824ddd53d9 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 23 Jun 2025 17:22:39 +0100 Subject: [PATCH 125/183] Reduce time filters - Reduce grid and cloud publishing filters to 1 year as we only care about quite recent data for those views. - Reduce filter for sync to 2 years as we should be sorting out publishing issues within that time frame. --- monitoring/publishing/db_update_sqlite.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monitoring/publishing/db_update_sqlite.py b/monitoring/publishing/db_update_sqlite.py index 81a597f..9f3098f 100644 --- a/monitoring/publishing/db_update_sqlite.py +++ b/monitoring/publishing/db_update_sqlite.py @@ -89,7 +89,7 @@ def refresh_gridsite(): Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries - WHERE LatestEndTime > DATE_SUB(NOW(), INTERVAL 3 YEAR) + WHERE LatestEndTime > DATE_SUB(NOW(), INTERVAL 1 YEAR) GROUP BY 1; """ fetchset = VSuperSummaries.objects.using('grid').raw(sql_query) @@ -119,7 +119,7 @@ def refresh_cloudsite(): SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords - WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 YEAR) + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 1 YEAR) GROUP BY SiteName ) AS a @@ -159,7 +159,7 @@ def refresh_gridsitesync(): MAX(LatestEndTime) AS RecordEnd FROM VSuperSummaries WHERE - Year >= YEAR(NOW()) - 3 AND + Year >= YEAR(NOW()) - 2 AND EarliestEndTime>'1900-01-01' AND LatestEndTime>'1900-01-01' GROUP BY @@ -175,7 +175,7 @@ def refresh_gridsitesync(): SUM(NumberOfJobs) AS RecordCountInDb FROM VSyncRecords WHERE - Year >= YEAR(NOW()) - 3 + Year >= YEAR(NOW()) - 2 GROUP BY Site, Year, Month; """ From 82e175dbc9129b73ffa27474648048f96fd4151c Mon Sep 17 00:00:00 2001 From: Sae126V Date: Thu, 14 Aug 2025 15:25:23 +0100 Subject: [PATCH 126/183] Move db_update_sqlite file to app level --- monitoring/{publishing => }/db_update_sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename monitoring/{publishing => }/db_update_sqlite.py (99%) diff --git a/monitoring/publishing/db_update_sqlite.py b/monitoring/db_update_sqlite.py similarity index 99% rename from monitoring/publishing/db_update_sqlite.py rename to monitoring/db_update_sqlite.py index 9f3098f..d608557 100644 --- a/monitoring/publishing/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -13,7 +13,7 @@ import pandas as pd -BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) # Find the root and the Django project sys.path.append(BASE_DIR) From 34631fb23c1a95f31a4caa308288168bf576e889 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 12 Aug 2025 10:32:37 +0100 Subject: [PATCH 127/183] Add benchmarks app --- monitoring/benchmarks/__init__.py | 0 monitoring/benchmarks/admin.py | 3 + monitoring/benchmarks/apps.py | 6 ++ monitoring/benchmarks/migrations/__init__.py | 0 monitoring/benchmarks/models.py | 13 ++++ monitoring/benchmarks/serializers.py | 21 +++++++ .../templates/benchmarksBySubmithost.html | 28 +++++++++ monitoring/benchmarks/tests.py | 3 + monitoring/benchmarks/urls.py | 17 +++++ monitoring/benchmarks/views.py | 63 +++++++++++++++++++ monitoring/settings.py | 1 + monitoring/urls.py | 1 + 12 files changed, 156 insertions(+) create mode 100644 monitoring/benchmarks/__init__.py create mode 100644 monitoring/benchmarks/admin.py create mode 100644 monitoring/benchmarks/apps.py create mode 100644 monitoring/benchmarks/migrations/__init__.py create mode 100644 monitoring/benchmarks/models.py create mode 100644 monitoring/benchmarks/serializers.py create mode 100644 monitoring/benchmarks/templates/benchmarksBySubmithost.html create mode 100644 monitoring/benchmarks/tests.py create mode 100644 monitoring/benchmarks/urls.py create mode 100644 monitoring/benchmarks/views.py diff --git a/monitoring/benchmarks/__init__.py b/monitoring/benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/benchmarks/admin.py b/monitoring/benchmarks/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/monitoring/benchmarks/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/monitoring/benchmarks/apps.py b/monitoring/benchmarks/apps.py new file mode 100644 index 0000000..9a0ebb8 --- /dev/null +++ b/monitoring/benchmarks/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class BenchmarksConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'benchmarks' diff --git a/monitoring/benchmarks/migrations/__init__.py b/monitoring/benchmarks/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/benchmarks/models.py b/monitoring/benchmarks/models.py new file mode 100644 index 0000000..c5c803e --- /dev/null +++ b/monitoring/benchmarks/models.py @@ -0,0 +1,13 @@ +from django.db import models + +class BenchmarksBySubmithost(models.Model): + fetched = models.DateTimeField(auto_now=True) + SiteName = models.CharField(max_length=255) + SubmitHost = models.CharField(max_length=255) + ServiceLevelType = models.DecimalField(max_digits=10, decimal_places=3) + ServiceLevel = models.CharField(max_length=50) + SourceView = models.CharField(max_length=50) + UpdateTime = models.DateTimeField() + + class Meta: + ordering = ('SiteName',) diff --git a/monitoring/benchmarks/serializers.py b/monitoring/benchmarks/serializers.py new file mode 100644 index 0000000..cf57253 --- /dev/null +++ b/monitoring/benchmarks/serializers.py @@ -0,0 +1,21 @@ +from rest_framework import serializers + +from monitoring.benchmarks.models import BenchmarksBySubmithost + + +class BenchmarksBySubmithostSerializer(serializers.HyperlinkedModelSerializer): + # Override default format with None so that Python datetime is used as + # ouput format. Encoding will be determined by the renderer and can be + # formatted by a template filter. + updated = serializers.DateTimeField(format=None) + + class Meta: + model = BenchmarksBySubmithost + fields = ( + 'SiteName', + 'SubmitHost', + 'ServiceLevelType', + 'ServiceLevel', + 'SourceView', + 'UpdateTime', + ) diff --git a/monitoring/benchmarks/templates/benchmarksBySubmithost.html b/monitoring/benchmarks/templates/benchmarksBySubmithost.html new file mode 100644 index 0000000..48f33e4 --- /dev/null +++ b/monitoring/benchmarks/templates/benchmarksBySubmithost.html @@ -0,0 +1,28 @@ + + + + + + + Sites publishing benchmark records + + + +

    Sites publishing benchmark records in last 2 months

    +

    Page last updated: {{ last_fetched|date:"Y-m-d H:i:s.u" }}

    +
    + + + + {% for site in sites %} + + + + + + + + + {% endfor %} +
    SiteSubmit hostService level typeService levelSource viewLast updated
    {{ benchmark.SiteName }}{{ benchmark.SubmitHost }}{{ benchmark.ServiceLevelType }}{{ benchmark.ServiceLevel }}{{ benchmark.SourceView }}{{ benchmark.UpdateTime|date:"Y-m-d H:i:s" }}
    + \ No newline at end of file diff --git a/monitoring/benchmarks/tests.py b/monitoring/benchmarks/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/monitoring/benchmarks/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/monitoring/benchmarks/urls.py b/monitoring/benchmarks/urls.py new file mode 100644 index 0000000..fa24785 --- /dev/null +++ b/monitoring/benchmarks/urls.py @@ -0,0 +1,17 @@ +from rest_framework import routers + +from monitoring.benchmarks import views +from django.urls import re_path + +router = routers.SimpleRouter() +router.register(r'benchmarks', views.BenchmarksViewSet) + +urlpatterns = [ + re_path( + r'^benchmarks/(?P[a-zA-Z0-9._-]+)/$', + views.BenchmarksViewSet.as_view({'get': 'retrieve'}), + name='site-benchmarks-detail' + ), +] + +urlpatterns += router.urls diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py new file mode 100644 index 0000000..511fbb6 --- /dev/null +++ b/monitoring/benchmarks/views.py @@ -0,0 +1,63 @@ +from django.shortcuts import render +from datetime import datetime, timedelta + +from django.db.models import Max +from django.shortcuts import get_object_or_404 +import pandas as pd + +from rest_framework import viewsets +from rest_framework.renderers import TemplateHTMLRenderer +from rest_framework.response import Response +from rest_framework.pagination import PageNumberPagination + + +from monitoring.benchmarks.models import BenchmarksBySubmithost + +from monitoring.benchmarks.serializers import BenchmarksBySubmithostSerializer + +class BenchmarksViewSet(viewsets.ReadOnlyModelViewSet): + queryset = BenchmarksBySubmithost.objects.all() + serializer_class = BenchmarksBySubmithostSerializer + template_name = 'benchmarksBySubmithost.html' + lookup_field = 'SiteName' + + def list(self, request): + last_fetched = BenchmarksBySubmithost.objects.aggregate(Max('fetched'))['fetched__max'] + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + + final_response = [] + response = super(BenchmarksViewSet, self).list(request) + + for single_dict in response.data: + date = single_dict.get('UpdateTime').replace(tzinfo=None) + # single_dict = update_dict_stdout_and_returncode(single_dict, date) + final_response.append(date) + + if type(request.accepted_renderer) is TemplateHTMLRenderer: + response.data = { + 'benchmark': final_response, + 'last_fetched': last_fetched + } + + return response + + # def retrieve(self, request, SiteName=None): + # last_fetched = BenchmarksBySubmithost.objects.aggregate(Max('fetched'))['fetched__max'] + # # If there's no data then last_fetched is None. + # if last_fetched is not None: + # print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + + # response = super(GridSiteViewSet, self).retrieve(request) + # date = response.data['updated'].replace(tzinfo=None) + # response.data = update_dict_stdout_and_returncode(response.data, date) + + # # Wrap data in a dict so that it can display in template. + # if type(request.accepted_renderer) is TemplateHTMLRenderer: + # # Single result put in list to work with same HTML template. + # response.data = { + # 'benchmark': [response.data], + # 'last_fetched': last_fetched + # } + + # return response diff --git a/monitoring/settings.py b/monitoring/settings.py index 627800e..e00bb9d 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -76,6 +76,7 @@ 'rest_framework', 'monitoring.publishing', 'monitoring.availability', + 'monitoring.benchmarks', ] REST_FRAMEWORK = { diff --git a/monitoring/urls.py b/monitoring/urls.py index 26e4055..7d08c4f 100644 --- a/monitoring/urls.py +++ b/monitoring/urls.py @@ -20,6 +20,7 @@ url(r'^admin/', admin.site.urls), url(r'^availability/', include('monitoring.availability.urls')), url(r'^publishing/', include('monitoring.publishing.urls')), + url(r'^benchmarks/', include('monitoring.benchmarks.urls')), url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), ] From 5e36e49d16c0454b0301e27ff2a7a379aebba51b Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 12 Aug 2025 11:12:19 +0100 Subject: [PATCH 128/183] Update app name --- monitoring/benchmarks/apps.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/monitoring/benchmarks/apps.py b/monitoring/benchmarks/apps.py index 9a0ebb8..bb0e0b3 100644 --- a/monitoring/benchmarks/apps.py +++ b/monitoring/benchmarks/apps.py @@ -2,5 +2,4 @@ class BenchmarksConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'benchmarks' + name = 'monitoring.benchmarks' From c1e7bada8802199946820dd0c2acd2e33768e625 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 12 Aug 2025 11:24:21 +0100 Subject: [PATCH 129/183] Add extra_kwargs --- monitoring/benchmarks/serializers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/monitoring/benchmarks/serializers.py b/monitoring/benchmarks/serializers.py index cf57253..80472a5 100644 --- a/monitoring/benchmarks/serializers.py +++ b/monitoring/benchmarks/serializers.py @@ -19,3 +19,8 @@ class Meta: 'SourceView', 'UpdateTime', ) + + # Sitename substitutes for pk + extra_kwargs = { + 'url': {'view_name': 'site-benchmarks-detail', 'lookup_field': 'SiteName'} + } From 9272d9b213eeb2dc2430b6e46b81154a71bf043e Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 12 Aug 2025 11:29:24 +0100 Subject: [PATCH 130/183] Update view name --- monitoring/benchmarks/serializers.py | 2 +- monitoring/benchmarks/urls.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/benchmarks/serializers.py b/monitoring/benchmarks/serializers.py index 80472a5..fd765ff 100644 --- a/monitoring/benchmarks/serializers.py +++ b/monitoring/benchmarks/serializers.py @@ -22,5 +22,5 @@ class Meta: # Sitename substitutes for pk extra_kwargs = { - 'url': {'view_name': 'site-benchmarks-detail', 'lookup_field': 'SiteName'} + 'url': {'view_name': 'benchmarksbysubmithost-list', 'lookup_field': 'SiteName'} } diff --git a/monitoring/benchmarks/urls.py b/monitoring/benchmarks/urls.py index fa24785..58b1839 100644 --- a/monitoring/benchmarks/urls.py +++ b/monitoring/benchmarks/urls.py @@ -10,7 +10,7 @@ re_path( r'^benchmarks/(?P[a-zA-Z0-9._-]+)/$', views.BenchmarksViewSet.as_view({'get': 'retrieve'}), - name='site-benchmarks-detail' + name='benchmarksbysubmithost-list' ), ] From 8e9703a13486cafc9e605d97218db5ffe4d4692d Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 12 Aug 2025 13:10:22 +0100 Subject: [PATCH 131/183] Rename view --- monitoring/benchmarks/serializers.py | 2 +- monitoring/benchmarks/urls.py | 2 +- monitoring/benchmarks/views.py | 60 ++++++++++++++++++---------- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/monitoring/benchmarks/serializers.py b/monitoring/benchmarks/serializers.py index fd765ff..0208f1e 100644 --- a/monitoring/benchmarks/serializers.py +++ b/monitoring/benchmarks/serializers.py @@ -22,5 +22,5 @@ class Meta: # Sitename substitutes for pk extra_kwargs = { - 'url': {'view_name': 'benchmarksbysubmithost-list', 'lookup_field': 'SiteName'} + 'url': {'view_name': 'benchmarksbysubmithost-details', 'lookup_field': 'SiteName'} } diff --git a/monitoring/benchmarks/urls.py b/monitoring/benchmarks/urls.py index 58b1839..24398ab 100644 --- a/monitoring/benchmarks/urls.py +++ b/monitoring/benchmarks/urls.py @@ -10,7 +10,7 @@ re_path( r'^benchmarks/(?P[a-zA-Z0-9._-]+)/$', views.BenchmarksViewSet.as_view({'get': 'retrieve'}), - name='benchmarksbysubmithost-list' + name='benchmarksbysubmithost-details' ), ] diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py index 511fbb6..fb922d7 100644 --- a/monitoring/benchmarks/views.py +++ b/monitoring/benchmarks/views.py @@ -15,10 +15,26 @@ from monitoring.benchmarks.serializers import BenchmarksBySubmithostSerializer +def update_dict_stdout_and_returncode(single_dict, date): + diff = datetime.today() - date + date = date.strftime("%Y-%m-%d") + + if diff <= timedelta(days=7): + single_dict['returncode'] = 0 + single_dict['stdout'] = "OK [ last published %s days ago: %s ]" % (diff.days, date) + elif diff > timedelta(days=7): + single_dict['returncode'] = 1 + single_dict['stdout'] = "WARNING [ last published %s days ago: %s ]" % (diff.days, date) + else: + single_dict['returncode'] = 3 + single_dict['stdout'] = "UNKNOWN" + return single_dict + + class BenchmarksViewSet(viewsets.ReadOnlyModelViewSet): queryset = BenchmarksBySubmithost.objects.all() serializer_class = BenchmarksBySubmithostSerializer - template_name = 'benchmarksBySubmithost.html' + template_name = 'benchmarksbysubmithost.html' lookup_field = 'SiteName' def list(self, request): @@ -31,8 +47,8 @@ def list(self, request): for single_dict in response.data: date = single_dict.get('UpdateTime').replace(tzinfo=None) - # single_dict = update_dict_stdout_and_returncode(single_dict, date) - final_response.append(date) + single_dict = update_dict_stdout_and_returncode(single_dict, date) + final_response.append(single_dict) if type(request.accepted_renderer) is TemplateHTMLRenderer: response.data = { @@ -42,22 +58,22 @@ def list(self, request): return response - # def retrieve(self, request, SiteName=None): - # last_fetched = BenchmarksBySubmithost.objects.aggregate(Max('fetched'))['fetched__max'] - # # If there's no data then last_fetched is None. - # if last_fetched is not None: - # print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - - # response = super(GridSiteViewSet, self).retrieve(request) - # date = response.data['updated'].replace(tzinfo=None) - # response.data = update_dict_stdout_and_returncode(response.data, date) - - # # Wrap data in a dict so that it can display in template. - # if type(request.accepted_renderer) is TemplateHTMLRenderer: - # # Single result put in list to work with same HTML template. - # response.data = { - # 'benchmark': [response.data], - # 'last_fetched': last_fetched - # } - - # return response + def retrieve(self, request, SiteName=None): + last_fetched = BenchmarksBySubmithost.objects.aggregate(Max('fetched'))['fetched__max'] + # If there's no data then last_fetched is None. + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + + response = super(BenchmarksViewSet, self).retrieve(request) + date = response.data['UpdateTime'].replace(tzinfo=None) + response.data = update_dict_stdout_and_returncode(response.data, date) + + # Wrap data in a dict so that it can display in template. + if type(request.accepted_renderer) is TemplateHTMLRenderer: + # Single result put in list to work with same HTML template. + response.data = { + 'benchmark': [response.data], + 'last_fetched': last_fetched + } + + return response From 07d9191b59d7328a12b8cfea68c607678486c4b1 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 12 Aug 2025 13:19:11 +0100 Subject: [PATCH 132/183] Update url --- monitoring/benchmarks/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/benchmarks/urls.py b/monitoring/benchmarks/urls.py index 24398ab..793ba20 100644 --- a/monitoring/benchmarks/urls.py +++ b/monitoring/benchmarks/urls.py @@ -4,11 +4,11 @@ from django.urls import re_path router = routers.SimpleRouter() -router.register(r'benchmarks', views.BenchmarksViewSet) +router.register('', views.BenchmarksViewSet) urlpatterns = [ re_path( - r'^benchmarks/(?P[a-zA-Z0-9._-]+)/$', + r'/(?P[a-zA-Z0-9._-]+)/$', views.BenchmarksViewSet.as_view({'get': 'retrieve'}), name='benchmarksbysubmithost-details' ), From d3c0a1cc0a3bdb2dc743db332b6d63f0ce0fd233 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 12 Aug 2025 13:38:03 +0100 Subject: [PATCH 133/183] Rename template --- ...enchmarksBySubmithost.html => benchmarks_by_submithost.html} | 2 +- monitoring/benchmarks/urls.py | 2 +- monitoring/benchmarks/views.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename monitoring/benchmarks/templates/{benchmarksBySubmithost.html => benchmarks_by_submithost.html} (96%) diff --git a/monitoring/benchmarks/templates/benchmarksBySubmithost.html b/monitoring/benchmarks/templates/benchmarks_by_submithost.html similarity index 96% rename from monitoring/benchmarks/templates/benchmarksBySubmithost.html rename to monitoring/benchmarks/templates/benchmarks_by_submithost.html index 48f33e4..31ccb12 100644 --- a/monitoring/benchmarks/templates/benchmarksBySubmithost.html +++ b/monitoring/benchmarks/templates/benchmarks_by_submithost.html @@ -14,7 +14,7 @@

    Sites publishing benchmark records in last 2 months

    SiteSubmit hostService level typeService levelSource viewLast updated - {% for site in sites %} + {% for benchmark in benchmarks %} {{ benchmark.SiteName }} {{ benchmark.SubmitHost }} diff --git a/monitoring/benchmarks/urls.py b/monitoring/benchmarks/urls.py index 793ba20..e41f0a4 100644 --- a/monitoring/benchmarks/urls.py +++ b/monitoring/benchmarks/urls.py @@ -8,7 +8,7 @@ urlpatterns = [ re_path( - r'/(?P[a-zA-Z0-9._-]+)/$', + r'^/(?P[a-zA-Z0-9._-]+)/$', views.BenchmarksViewSet.as_view({'get': 'retrieve'}), name='benchmarksbysubmithost-details' ), diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py index fb922d7..991dcc1 100644 --- a/monitoring/benchmarks/views.py +++ b/monitoring/benchmarks/views.py @@ -34,7 +34,7 @@ def update_dict_stdout_and_returncode(single_dict, date): class BenchmarksViewSet(viewsets.ReadOnlyModelViewSet): queryset = BenchmarksBySubmithost.objects.all() serializer_class = BenchmarksBySubmithostSerializer - template_name = 'benchmarksbysubmithost.html' + template_name = 'benchmarks_by_submithost.html' lookup_field = 'SiteName' def list(self, request): From 76ad09e4844e9969880d75ccdb4433fb1d732824 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 12 Aug 2025 14:56:13 +0100 Subject: [PATCH 134/183] Refresh benchmarks data --- monitoring/benchmarks/models.py | 33 ++++++++++++++++++++++++++++ monitoring/db_update_sqlite.py | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/monitoring/benchmarks/models.py b/monitoring/benchmarks/models.py index c5c803e..00fe24c 100644 --- a/monitoring/benchmarks/models.py +++ b/monitoring/benchmarks/models.py @@ -11,3 +11,36 @@ class BenchmarksBySubmithost(models.Model): class Meta: ordering = ('SiteName',) + +class VJobRecords(models.Model): + Site = models.CharField(max_length=255, primary_key=True) + SubmitHost = models.CharField(max_length=255) + ServiceLevelType = models.DecimalField(max_digits=10, decimal_places=3) + ServiceLevel = models.CharField(max_length=50) + UpdateTime = models.DateTimeField() + + class Meta: + managed = False + db_table = 'VJobRecords' + +class VSummaries(models.Model): + Site = models.CharField(max_length=255, primary_key=True) + SubmitHost = models.CharField(max_length=255) + ServiceLevelType = models.DecimalField(max_digits=10, decimal_places=3) + ServiceLevel = models.CharField(max_length=50) + UpdateTime = models.DateTimeField() + + class Meta: + managed = False + db_table = 'VSummaries' + +class VNormalisedSummaries(models.Model): + Site = models.CharField(max_length=255, primary_key=True) + SubmitHost = models.CharField(max_length=255) + ServiceLevelType = models.DecimalField(max_digits=10, decimal_places=3) + ServiceLevel = models.CharField(max_length=50) + UpdateTime = models.DateTimeField() + + class Meta: + managed = False + db_table = 'VNormalisedSummaries' diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index d608557..9a1d9a2 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -45,6 +45,12 @@ get_year_month_str ) +from monitoring.benchmarks.models import ( + BenchmarksBySubmithost, + VJobRecords, + VSummaries, + VNormalisedSummaries, +) try: # Read configuration from the file @@ -227,6 +233,38 @@ def refresh_gridsitesync(): except DatabaseError: log.exception('Error while trying to refresh GridSiteSync') +def refresh_BenchmarksBySubmitHost(): + try: + sql_query = """ + SELECT + Site, + SubmitHost, + ServiceLevelType, + ServiceLevel, + max(UpdateTime) AS LatestPublish + FROM VJobRecords + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 2 MONTH) + GROUP BY 1; + """ + fetchset = VJobRecords.objects.raw(sql_query) + + for f in fetchset: + BenchmarksBySubmithost.objects.update_or_create( + defaults={ + 'UpdateTime': f.LatestPublish, + 'SourceView': 'VJobRecords' + }, + SiteName=f.Site, + SubmitHost=f.SubmitHost, + ServiceLevelType=f.ServiceLevelType, + ServiceLevel=f.ServiceLevel, + ) + + log.info("Refreshed BenchmarksBySubmitHost") + + except DatabaseError: + log.exception('Error while trying to refresh BenchmarksBySubmitHost') + if __name__ == "__main__": log.info('=====================') @@ -234,6 +272,7 @@ def refresh_gridsitesync(): refresh_gridsite() refresh_cloudsite() refresh_gridsitesync() + refresh_BenchmarksBySubmitHost() log.info( "Data retrieval and processing attempted. " From 43f18bbcc8a81273bbf6d2360b31e6b7184a21d4 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Wed, 13 Aug 2025 16:28:15 +0100 Subject: [PATCH 135/183] Fetch data from VSummaries --- monitoring/benchmarks/models.py | 1 + monitoring/db_update_sqlite.py | 23 ++++++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/monitoring/benchmarks/models.py b/monitoring/benchmarks/models.py index 00fe24c..778f630 100644 --- a/monitoring/benchmarks/models.py +++ b/monitoring/benchmarks/models.py @@ -18,6 +18,7 @@ class VJobRecords(models.Model): ServiceLevelType = models.DecimalField(max_digits=10, decimal_places=3) ServiceLevel = models.CharField(max_length=50) UpdateTime = models.DateTimeField() + EndTime = models.DateTimeField() class Meta: managed = False diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 9a1d9a2..afbbf93 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -235,6 +235,19 @@ def refresh_gridsitesync(): def refresh_BenchmarksBySubmitHost(): try: + # sql_query = """ + # SELECT + # Site, + # SubmitHost, + # ServiceLevelType, + # ServiceLevel, + # max(UpdateTime) AS LatestPublish + # FROM VJobRecords + # WHERE EndTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + # AND UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + # GROUP BY Site, SubmitHost; + # """ + # fetchset = VJobRecords.objects.raw(sql_query) sql_query = """ SELECT Site, @@ -242,17 +255,17 @@ def refresh_BenchmarksBySubmitHost(): ServiceLevelType, ServiceLevel, max(UpdateTime) AS LatestPublish - FROM VJobRecords - WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 2 MONTH) - GROUP BY 1; + FROM VSummaries + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + GROUP BY Site, SubmitHost; """ - fetchset = VJobRecords.objects.raw(sql_query) + fetchset = VSummaries.objects.raw(sql_query) for f in fetchset: BenchmarksBySubmithost.objects.update_or_create( defaults={ 'UpdateTime': f.LatestPublish, - 'SourceView': 'VJobRecords' + 'SourceView': 'VSummaries' }, SiteName=f.Site, SubmitHost=f.SubmitHost, From 0ccdf751096eb82a91a07d778d487f65d32dfaeb Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Thu, 14 Aug 2025 16:37:41 +0100 Subject: [PATCH 136/183] Use grid database --- monitoring/db_update_sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index afbbf93..e71547c 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -259,7 +259,7 @@ def refresh_BenchmarksBySubmitHost(): WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) GROUP BY Site, SubmitHost; """ - fetchset = VSummaries.objects.raw(sql_query) + fetchset = VSummaries.objects.using('grid').raw(sql_query) for f in fetchset: BenchmarksBySubmithost.objects.update_or_create( From 281993adaab646da76e2e30a57fb9c9dcf292144 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Thu, 14 Aug 2025 17:20:01 +0100 Subject: [PATCH 137/183] Fix the datatype --- monitoring/benchmarks/models.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/monitoring/benchmarks/models.py b/monitoring/benchmarks/models.py index 778f630..5ee40be 100644 --- a/monitoring/benchmarks/models.py +++ b/monitoring/benchmarks/models.py @@ -4,8 +4,8 @@ class BenchmarksBySubmithost(models.Model): fetched = models.DateTimeField(auto_now=True) SiteName = models.CharField(max_length=255) SubmitHost = models.CharField(max_length=255) - ServiceLevelType = models.DecimalField(max_digits=10, decimal_places=3) - ServiceLevel = models.CharField(max_length=50) + ServiceLevelType = models.CharField(max_length=50) + ServiceLevel = models.DecimalField(max_digits=10, decimal_places=3) SourceView = models.CharField(max_length=50) UpdateTime = models.DateTimeField() @@ -15,8 +15,8 @@ class Meta: class VJobRecords(models.Model): Site = models.CharField(max_length=255, primary_key=True) SubmitHost = models.CharField(max_length=255) - ServiceLevelType = models.DecimalField(max_digits=10, decimal_places=3) - ServiceLevel = models.CharField(max_length=50) + ServiceLevelType = models.CharField(max_length=50) + ServiceLevel = models.DecimalField(max_digits=10, decimal_places=3) UpdateTime = models.DateTimeField() EndTime = models.DateTimeField() @@ -27,8 +27,8 @@ class Meta: class VSummaries(models.Model): Site = models.CharField(max_length=255, primary_key=True) SubmitHost = models.CharField(max_length=255) - ServiceLevelType = models.DecimalField(max_digits=10, decimal_places=3) - ServiceLevel = models.CharField(max_length=50) + ServiceLevelType = models.CharField(max_length=50) + ServiceLevel = models.DecimalField(max_digits=10, decimal_places=3) UpdateTime = models.DateTimeField() class Meta: @@ -38,8 +38,8 @@ class Meta: class VNormalisedSummaries(models.Model): Site = models.CharField(max_length=255, primary_key=True) SubmitHost = models.CharField(max_length=255) - ServiceLevelType = models.DecimalField(max_digits=10, decimal_places=3) - ServiceLevel = models.CharField(max_length=50) + ServiceLevelType = models.CharField(max_length=50) + ServiceLevel = models.DecimalField(max_digits=10, decimal_places=3) UpdateTime = models.DateTimeField() class Meta: From b70d9fb9546d469d19e87ed6081f410d649954a4 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Thu, 14 Aug 2025 21:57:55 +0100 Subject: [PATCH 138/183] Update variable name --- monitoring/benchmarks/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/benchmarks/serializers.py b/monitoring/benchmarks/serializers.py index 0208f1e..3dc31bd 100644 --- a/monitoring/benchmarks/serializers.py +++ b/monitoring/benchmarks/serializers.py @@ -7,7 +7,7 @@ class BenchmarksBySubmithostSerializer(serializers.HyperlinkedModelSerializer): # Override default format with None so that Python datetime is used as # ouput format. Encoding will be determined by the renderer and can be # formatted by a template filter. - updated = serializers.DateTimeField(format=None) + UpdateTime = serializers.DateTimeField(format=None) class Meta: model = BenchmarksBySubmithost From 31e499692753fb7b3cfdff2c79fa859276a1544e Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Thu, 14 Aug 2025 22:09:55 +0100 Subject: [PATCH 139/183] Update variable name --- monitoring/benchmarks/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py index 991dcc1..42cb9d9 100644 --- a/monitoring/benchmarks/views.py +++ b/monitoring/benchmarks/views.py @@ -52,7 +52,7 @@ def list(self, request): if type(request.accepted_renderer) is TemplateHTMLRenderer: response.data = { - 'benchmark': final_response, + 'benchmarks': final_response, 'last_fetched': last_fetched } From 1f380d50ee2269fa5e6da822b7ecf4afa1b3dd4b Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Fri, 15 Aug 2025 12:02:38 +0100 Subject: [PATCH 140/183] Fetch data from two tables --- monitoring/benchmarks/views.py | 37 ++++++++++--------------------- monitoring/db_update_sqlite.py | 40 ++++++++++++++++------------------ 2 files changed, 31 insertions(+), 46 deletions(-) diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py index 42cb9d9..cb1d3fe 100644 --- a/monitoring/benchmarks/views.py +++ b/monitoring/benchmarks/views.py @@ -15,22 +15,6 @@ from monitoring.benchmarks.serializers import BenchmarksBySubmithostSerializer -def update_dict_stdout_and_returncode(single_dict, date): - diff = datetime.today() - date - date = date.strftime("%Y-%m-%d") - - if diff <= timedelta(days=7): - single_dict['returncode'] = 0 - single_dict['stdout'] = "OK [ last published %s days ago: %s ]" % (diff.days, date) - elif diff > timedelta(days=7): - single_dict['returncode'] = 1 - single_dict['stdout'] = "WARNING [ last published %s days ago: %s ]" % (diff.days, date) - else: - single_dict['returncode'] = 3 - single_dict['stdout'] = "UNKNOWN" - return single_dict - - class BenchmarksViewSet(viewsets.ReadOnlyModelViewSet): queryset = BenchmarksBySubmithost.objects.all() serializer_class = BenchmarksBySubmithostSerializer @@ -47,8 +31,7 @@ def list(self, request): for single_dict in response.data: date = single_dict.get('UpdateTime').replace(tzinfo=None) - single_dict = update_dict_stdout_and_returncode(single_dict, date) - final_response.append(single_dict) + final_response.append(date) if type(request.accepted_renderer) is TemplateHTMLRenderer: response.data = { @@ -63,17 +46,21 @@ def retrieve(self, request, SiteName=None): # If there's no data then last_fetched is None. if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - - response = super(BenchmarksViewSet, self).retrieve(request) - date = response.data['UpdateTime'].replace(tzinfo=None) - response.data = update_dict_stdout_and_returncode(response.data, date) + + final_response = [] + sites_list_qs = BenchmarksBySubmithost.objects.filter(SiteName=SiteName) + sites_list_serializer = self.get_serializer(sites_list_qs, many=True) + + for single_dict in sites_list_serializer.data: + date = single_dict.get('UpdateTime').replace(tzinfo=None) + final_response.append(date) # Wrap data in a dict so that it can display in template. if type(request.accepted_renderer) is TemplateHTMLRenderer: # Single result put in list to work with same HTML template. - response.data = { - 'benchmark': [response.data], + response = { + 'benchmarks': final_response, 'last_fetched': last_fetched } - return response + return Response(response) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index e71547c..02780f6 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -233,50 +233,48 @@ def refresh_gridsitesync(): except DatabaseError: log.exception('Error while trying to refresh GridSiteSync') + def refresh_BenchmarksBySubmitHost(): + # views = ['VSummaries', 'VJobRecords', 'VNormalisedSummaries'] + views = ['VSummaries', 'VNormalisedSummaries'] + for view in views: + refresh_BenchmarksBySubmitHost_from_view(view) + + +def refresh_BenchmarksBySubmitHost_from_view(view_name): try: - # sql_query = """ - # SELECT - # Site, - # SubmitHost, - # ServiceLevelType, - # ServiceLevel, - # max(UpdateTime) AS LatestPublish - # FROM VJobRecords - # WHERE EndTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) - # AND UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) - # GROUP BY Site, SubmitHost; - # """ - # fetchset = VJobRecords.objects.raw(sql_query) - sql_query = """ + sql_query = f""" SELECT Site, SubmitHost, ServiceLevelType, ServiceLevel, max(UpdateTime) AS LatestPublish - FROM VSummaries + FROM {view_name} WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) GROUP BY Site, SubmitHost; """ - fetchset = VSummaries.objects.using('grid').raw(sql_query) + + # Dynamically get the model class from globals + model_class = globals()[view_name] + fetchset = model_class.objects.using('grid').raw(sql_query) for f in fetchset: BenchmarksBySubmithost.objects.update_or_create( defaults={ 'UpdateTime': f.LatestPublish, - 'SourceView': 'VSummaries' - }, + 'SourceView': view_name + }, SiteName=f.Site, SubmitHost=f.SubmitHost, ServiceLevelType=f.ServiceLevelType, ServiceLevel=f.ServiceLevel, ) - log.info("Refreshed BenchmarksBySubmitHost") + log.info(f"Refreshed BenchmarksBySubmitHost from {view_name}") - except DatabaseError: - log.exception('Error while trying to refresh BenchmarksBySubmitHost') + except Exception: + log.exception(f'Error while trying to refresh BenchmarksBySubmitHost from {view_name}') if __name__ == "__main__": From 736261facc0e4dda7ac7cff6f7819be193cfaad3 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Fri, 15 Aug 2025 13:45:37 +0100 Subject: [PATCH 141/183] Refactoring --- monitoring/benchmarks/views.py | 15 ++------------- monitoring/db_update_sqlite.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py index cb1d3fe..62afd95 100644 --- a/monitoring/benchmarks/views.py +++ b/monitoring/benchmarks/views.py @@ -26,16 +26,11 @@ def list(self, request): if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - final_response = [] response = super(BenchmarksViewSet, self).list(request) - for single_dict in response.data: - date = single_dict.get('UpdateTime').replace(tzinfo=None) - final_response.append(date) - if type(request.accepted_renderer) is TemplateHTMLRenderer: response.data = { - 'benchmarks': final_response, + 'benchmarks': response.data, 'last_fetched': last_fetched } @@ -47,19 +42,13 @@ def retrieve(self, request, SiteName=None): if last_fetched is not None: print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - final_response = [] sites_list_qs = BenchmarksBySubmithost.objects.filter(SiteName=SiteName) sites_list_serializer = self.get_serializer(sites_list_qs, many=True) - for single_dict in sites_list_serializer.data: - date = single_dict.get('UpdateTime').replace(tzinfo=None) - final_response.append(date) - # Wrap data in a dict so that it can display in template. if type(request.accepted_renderer) is TemplateHTMLRenderer: - # Single result put in list to work with same HTML template. response = { - 'benchmarks': final_response, + 'benchmarks': sites_list_serializer, 'last_fetched': last_fetched } diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 02780f6..cebc24e 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -242,8 +242,21 @@ def refresh_BenchmarksBySubmitHost(): def refresh_BenchmarksBySubmitHost_from_view(view_name): - try: - sql_query = f""" + try: + if view_name == 'VSummaries': + sql_query = f""" + SELECT + Site, + SubmitHost, + ServiceLevelType, + ServiceLevel, + max(UpdateTime) AS LatestPublish + FROM {view_name} + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + GROUP BY Site, SubmitHost; + """ + elif view_name == 'VJobRecords': + sql_query = f""" SELECT Site, SubmitHost, @@ -254,6 +267,21 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) GROUP BY Site, SubmitHost; """ + elif view_name == 'VNormalisedSummaries': + sql_query = f""" + SELECT + Site, + SubmitHost, + ServiceLevelType, + CpuDuration AS ServiceLevel, + max(UpdateTime) AS LatestPublish + FROM {view_name} + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + GROUP BY Site, SubmitHost; + """ + else: + log.warning(f"Unknown view name: {view_name}") + return # Dynamically get the model class from globals model_class = globals()[view_name] From 85a493b1c175d23b3d00b1cb20975ef98c9255f6 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Fri, 15 Aug 2025 14:31:12 +0100 Subject: [PATCH 142/183] Update select query for VNormalisedSummaries --- monitoring/db_update_sqlite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index cebc24e..5948dac 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -273,10 +273,11 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): Site, SubmitHost, ServiceLevelType, - CpuDuration AS ServiceLevel, + (NormalisedWallDuration / WallDuration) AS ServiceLevel, max(UpdateTime) AS LatestPublish FROM {view_name} WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + AND WallDuration > 0 GROUP BY Site, SubmitHost; """ else: From 6798431118a905ab9dc8e81013b3cf615ef6c0b2 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Mon, 18 Aug 2025 13:50:37 +0100 Subject: [PATCH 143/183] Refactoring --- .../templates/benchmarks_by_submithost.html | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/monitoring/benchmarks/templates/benchmarks_by_submithost.html b/monitoring/benchmarks/templates/benchmarks_by_submithost.html index 31ccb12..e8a3c96 100644 --- a/monitoring/benchmarks/templates/benchmarks_by_submithost.html +++ b/monitoring/benchmarks/templates/benchmarks_by_submithost.html @@ -1,9 +1,8 @@ - - + Sites publishing benchmark records @@ -12,7 +11,12 @@

    Sites publishing benchmark records in last 2 months

    Page last updated: {{ last_fetched|date:"Y-m-d H:i:s.u" }}

    - + + + + + + {% for benchmark in benchmarks %} @@ -25,4 +29,6 @@

    Sites publishing benchmark records in last 2 months

    {% endfor %}
    SiteSubmit hostService level typeService levelSource viewLast updatedSiteSubmit hostService level typeService levelSource viewLast updated
    - \ No newline at end of file + + + From d6d0c94142ce432d783da06d49ff56aee25f340b Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 12 Aug 2025 14:56:13 +0100 Subject: [PATCH 144/183] Refresh benchmarks data --- monitoring/db_update_sqlite.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 5948dac..9c64c71 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -233,6 +233,38 @@ def refresh_gridsitesync(): except DatabaseError: log.exception('Error while trying to refresh GridSiteSync') +def refresh_BenchmarksBySubmitHost(): + try: + sql_query = """ + SELECT + Site, + SubmitHost, + ServiceLevelType, + ServiceLevel, + max(UpdateTime) AS LatestPublish + FROM VJobRecords + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 2 MONTH) + GROUP BY 1; + """ + fetchset = VJobRecords.objects.raw(sql_query) + + for f in fetchset: + BenchmarksBySubmithost.objects.update_or_create( + defaults={ + 'UpdateTime': f.LatestPublish, + 'SourceView': 'VJobRecords' + }, + SiteName=f.Site, + SubmitHost=f.SubmitHost, + ServiceLevelType=f.ServiceLevelType, + ServiceLevel=f.ServiceLevel, + ) + + log.info("Refreshed BenchmarksBySubmitHost") + + except DatabaseError: + log.exception('Error while trying to refresh BenchmarksBySubmitHost') + def refresh_BenchmarksBySubmitHost(): # views = ['VSummaries', 'VJobRecords', 'VNormalisedSummaries'] From 2c51440d584179cdec520385088675fa9bfea912 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Wed, 13 Aug 2025 16:28:15 +0100 Subject: [PATCH 145/183] Fetch data from VSummaries --- monitoring/db_update_sqlite.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 9c64c71..7192bee 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -235,6 +235,19 @@ def refresh_gridsitesync(): def refresh_BenchmarksBySubmitHost(): try: + # sql_query = """ + # SELECT + # Site, + # SubmitHost, + # ServiceLevelType, + # ServiceLevel, + # max(UpdateTime) AS LatestPublish + # FROM VJobRecords + # WHERE EndTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + # AND UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + # GROUP BY Site, SubmitHost; + # """ + # fetchset = VJobRecords.objects.raw(sql_query) sql_query = """ SELECT Site, @@ -242,17 +255,17 @@ def refresh_BenchmarksBySubmitHost(): ServiceLevelType, ServiceLevel, max(UpdateTime) AS LatestPublish - FROM VJobRecords - WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 2 MONTH) - GROUP BY 1; + FROM VSummaries + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + GROUP BY Site, SubmitHost; """ - fetchset = VJobRecords.objects.raw(sql_query) + fetchset = VSummaries.objects.raw(sql_query) for f in fetchset: BenchmarksBySubmithost.objects.update_or_create( defaults={ 'UpdateTime': f.LatestPublish, - 'SourceView': 'VJobRecords' + 'SourceView': 'VSummaries' }, SiteName=f.Site, SubmitHost=f.SubmitHost, From d3bff6339c747a4bb798034bba91b92f779098c1 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Thu, 14 Aug 2025 16:37:41 +0100 Subject: [PATCH 146/183] Use grid database --- monitoring/db_update_sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 7192bee..002295f 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -259,7 +259,7 @@ def refresh_BenchmarksBySubmitHost(): WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) GROUP BY Site, SubmitHost; """ - fetchset = VSummaries.objects.raw(sql_query) + fetchset = VSummaries.objects.using('grid').raw(sql_query) for f in fetchset: BenchmarksBySubmithost.objects.update_or_create( From 3917a3127980a3ee0a4766fa50f0f0bde0532f5d Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Fri, 15 Aug 2025 12:02:38 +0100 Subject: [PATCH 147/183] Fetch data from two tables --- monitoring/db_update_sqlite.py | 40 ++++++++++++++++------------------ 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 002295f..714d5f1 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -233,50 +233,48 @@ def refresh_gridsitesync(): except DatabaseError: log.exception('Error while trying to refresh GridSiteSync') + def refresh_BenchmarksBySubmitHost(): + # views = ['VSummaries', 'VJobRecords', 'VNormalisedSummaries'] + views = ['VSummaries', 'VNormalisedSummaries'] + for view in views: + refresh_BenchmarksBySubmitHost_from_view(view) + + +def refresh_BenchmarksBySubmitHost_from_view(view_name): try: - # sql_query = """ - # SELECT - # Site, - # SubmitHost, - # ServiceLevelType, - # ServiceLevel, - # max(UpdateTime) AS LatestPublish - # FROM VJobRecords - # WHERE EndTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) - # AND UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) - # GROUP BY Site, SubmitHost; - # """ - # fetchset = VJobRecords.objects.raw(sql_query) - sql_query = """ + sql_query = f""" SELECT Site, SubmitHost, ServiceLevelType, ServiceLevel, max(UpdateTime) AS LatestPublish - FROM VSummaries + FROM {view_name} WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) GROUP BY Site, SubmitHost; """ - fetchset = VSummaries.objects.using('grid').raw(sql_query) + + # Dynamically get the model class from globals + model_class = globals()[view_name] + fetchset = model_class.objects.using('grid').raw(sql_query) for f in fetchset: BenchmarksBySubmithost.objects.update_or_create( defaults={ 'UpdateTime': f.LatestPublish, - 'SourceView': 'VSummaries' - }, + 'SourceView': view_name + }, SiteName=f.Site, SubmitHost=f.SubmitHost, ServiceLevelType=f.ServiceLevelType, ServiceLevel=f.ServiceLevel, ) - log.info("Refreshed BenchmarksBySubmitHost") + log.info(f"Refreshed BenchmarksBySubmitHost from {view_name}") - except DatabaseError: - log.exception('Error while trying to refresh BenchmarksBySubmitHost') + except Exception: + log.exception(f'Error while trying to refresh BenchmarksBySubmitHost from {view_name}') def refresh_BenchmarksBySubmitHost(): From eb5e1b09203e971d63f28cd55b7bc504f3b14452 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Fri, 15 Aug 2025 13:45:37 +0100 Subject: [PATCH 148/183] Refactoring --- monitoring/db_update_sqlite.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 714d5f1..e120efd 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -242,8 +242,21 @@ def refresh_BenchmarksBySubmitHost(): def refresh_BenchmarksBySubmitHost_from_view(view_name): - try: - sql_query = f""" + try: + if view_name == 'VSummaries': + sql_query = f""" + SELECT + Site, + SubmitHost, + ServiceLevelType, + ServiceLevel, + max(UpdateTime) AS LatestPublish + FROM {view_name} + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + GROUP BY Site, SubmitHost; + """ + elif view_name == 'VJobRecords': + sql_query = f""" SELECT Site, SubmitHost, @@ -254,6 +267,21 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) GROUP BY Site, SubmitHost; """ + elif view_name == 'VNormalisedSummaries': + sql_query = f""" + SELECT + Site, + SubmitHost, + ServiceLevelType, + CpuDuration AS ServiceLevel, + max(UpdateTime) AS LatestPublish + FROM {view_name} + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + GROUP BY Site, SubmitHost; + """ + else: + log.warning(f"Unknown view name: {view_name}") + return # Dynamically get the model class from globals model_class = globals()[view_name] From a477d2b77ef849c07509b39333e186500599c3e9 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Fri, 15 Aug 2025 14:31:12 +0100 Subject: [PATCH 149/183] Update select query for VNormalisedSummaries --- monitoring/db_update_sqlite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index e120efd..4c98816 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -273,10 +273,11 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): Site, SubmitHost, ServiceLevelType, - CpuDuration AS ServiceLevel, + (NormalisedWallDuration / WallDuration) AS ServiceLevel, max(UpdateTime) AS LatestPublish FROM {view_name} WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + AND WallDuration > 0 GROUP BY Site, SubmitHost; """ else: From f8cd8e9dc1914178472582badeb6638204db13ab Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Mon, 18 Aug 2025 16:59:15 +0100 Subject: [PATCH 150/183] Add verbose_name --- monitoring/benchmarks/models.py | 5 ++++- monitoring/benchmarks/serializers.py | 5 ----- monitoring/benchmarks/urls.py | 16 +++------------- monitoring/benchmarks/views.py | 22 ---------------------- monitoring/db_update_sqlite.py | 2 +- 5 files changed, 8 insertions(+), 42 deletions(-) diff --git a/monitoring/benchmarks/models.py b/monitoring/benchmarks/models.py index 5ee40be..afc51e1 100644 --- a/monitoring/benchmarks/models.py +++ b/monitoring/benchmarks/models.py @@ -23,6 +23,7 @@ class VJobRecords(models.Model): class Meta: managed = False db_table = 'VJobRecords' + verbose_name = 'Job Record' class VSummaries(models.Model): Site = models.CharField(max_length=255, primary_key=True) @@ -34,6 +35,7 @@ class VSummaries(models.Model): class Meta: managed = False db_table = 'VSummaries' + verbose_name = 'Summary' class VNormalisedSummaries(models.Model): Site = models.CharField(max_length=255, primary_key=True) @@ -44,4 +46,5 @@ class VNormalisedSummaries(models.Model): class Meta: managed = False - db_table = 'VNormalisedSummaries' + db_table = 'VNormalisedSummaries' + verbose_name = 'Normalised Summary' diff --git a/monitoring/benchmarks/serializers.py b/monitoring/benchmarks/serializers.py index 3dc31bd..0c66725 100644 --- a/monitoring/benchmarks/serializers.py +++ b/monitoring/benchmarks/serializers.py @@ -19,8 +19,3 @@ class Meta: 'SourceView', 'UpdateTime', ) - - # Sitename substitutes for pk - extra_kwargs = { - 'url': {'view_name': 'benchmarksbysubmithost-details', 'lookup_field': 'SiteName'} - } diff --git a/monitoring/benchmarks/urls.py b/monitoring/benchmarks/urls.py index e41f0a4..40dff78 100644 --- a/monitoring/benchmarks/urls.py +++ b/monitoring/benchmarks/urls.py @@ -1,17 +1,7 @@ -from rest_framework import routers +from django.urls import path from monitoring.benchmarks import views -from django.urls import re_path - -router = routers.SimpleRouter() -router.register('', views.BenchmarksViewSet) urlpatterns = [ - re_path( - r'^/(?P[a-zA-Z0-9._-]+)/$', - views.BenchmarksViewSet.as_view({'get': 'retrieve'}), - name='benchmarksbysubmithost-details' - ), -] - -urlpatterns += router.urls + path('', views.BenchmarksViewSet), +] \ No newline at end of file diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py index 62afd95..8b401b4 100644 --- a/monitoring/benchmarks/views.py +++ b/monitoring/benchmarks/views.py @@ -2,13 +2,9 @@ from datetime import datetime, timedelta from django.db.models import Max -from django.shortcuts import get_object_or_404 -import pandas as pd from rest_framework import viewsets from rest_framework.renderers import TemplateHTMLRenderer -from rest_framework.response import Response -from rest_framework.pagination import PageNumberPagination from monitoring.benchmarks.models import BenchmarksBySubmithost @@ -35,21 +31,3 @@ def list(self, request): } return response - - def retrieve(self, request, SiteName=None): - last_fetched = BenchmarksBySubmithost.objects.aggregate(Max('fetched'))['fetched__max'] - # If there's no data then last_fetched is None. - if last_fetched is not None: - print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) - - sites_list_qs = BenchmarksBySubmithost.objects.filter(SiteName=SiteName) - sites_list_serializer = self.get_serializer(sites_list_qs, many=True) - - # Wrap data in a dict so that it can display in template. - if type(request.accepted_renderer) is TemplateHTMLRenderer: - response = { - 'benchmarks': sites_list_serializer, - 'last_fetched': last_fetched - } - - return Response(response) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 4c98816..18452d5 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -292,7 +292,7 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): BenchmarksBySubmithost.objects.update_or_create( defaults={ 'UpdateTime': f.LatestPublish, - 'SourceView': view_name + 'SourceView': model_class._meta.verbose_name }, SiteName=f.Site, SubmitHost=f.SubmitHost, From 9516de8e8fa6f537d49e21fe8ed1224564dcd980 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Mon, 18 Aug 2025 21:18:57 +0100 Subject: [PATCH 151/183] Fix the url --- monitoring/benchmarks/urls.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/monitoring/benchmarks/urls.py b/monitoring/benchmarks/urls.py index 40dff78..8f3b790 100644 --- a/monitoring/benchmarks/urls.py +++ b/monitoring/benchmarks/urls.py @@ -1,7 +1,11 @@ -from django.urls import path +from rest_framework import routers +from django.urls import path, include from monitoring.benchmarks import views +router = routers.SimpleRouter() +router.register('', views.BenchmarksViewSet) + urlpatterns = [ - path('', views.BenchmarksViewSet), + path('', include(router.urls)), ] \ No newline at end of file From a5389c32e7fd41038e341f9b1600d31db57ca936 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 19 Aug 2025 09:11:39 +0100 Subject: [PATCH 152/183] Update query for VJobRecords --- monitoring/db_update_sqlite.py | 92 +++------------------------------- 1 file changed, 8 insertions(+), 84 deletions(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 18452d5..4333249 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -11,6 +11,7 @@ import django from django.db import DatabaseError import pandas as pd +from django.utils.timezone import make_aware, is_naive BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) @@ -45,12 +46,7 @@ get_year_month_str ) -from monitoring.benchmarks.models import ( - BenchmarksBySubmithost, - VJobRecords, - VSummaries, - VNormalisedSummaries, -) +from monitoring.benchmarks.models import BenchmarksBySubmithost try: # Read configuration from the file @@ -235,8 +231,7 @@ def refresh_gridsitesync(): def refresh_BenchmarksBySubmitHost(): - # views = ['VSummaries', 'VJobRecords', 'VNormalisedSummaries'] - views = ['VSummaries', 'VNormalisedSummaries'] + views = ['VSummaries', 'VJobRecords', 'VNormalisedSummaries'] for view in views: refresh_BenchmarksBySubmitHost_from_view(view) @@ -264,7 +259,8 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): ServiceLevel, max(UpdateTime) AS LatestPublish FROM {view_name} - WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + WHERE EndTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + AND UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) GROUP BY Site, SubmitHost; """ elif view_name == 'VNormalisedSummaries': @@ -291,7 +287,7 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): for f in fetchset: BenchmarksBySubmithost.objects.update_or_create( defaults={ - 'UpdateTime': f.LatestPublish, + 'UpdateTime': make_aware(f.LatestPublish) if is_naive(f.LatestPublish) else f.LatestPublish, 'SourceView': model_class._meta.verbose_name }, SiteName=f.Site, @@ -303,79 +299,7 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): log.info(f"Refreshed BenchmarksBySubmitHost from {view_name}") except Exception: - log.exception(f'Error while trying to refresh BenchmarksBySubmitHost from {view_name}') - - -def refresh_BenchmarksBySubmitHost(): - # views = ['VSummaries', 'VJobRecords', 'VNormalisedSummaries'] - views = ['VSummaries', 'VNormalisedSummaries'] - for view in views: - refresh_BenchmarksBySubmitHost_from_view(view) - - -def refresh_BenchmarksBySubmitHost_from_view(view_name): - try: - if view_name == 'VSummaries': - sql_query = f""" - SELECT - Site, - SubmitHost, - ServiceLevelType, - ServiceLevel, - max(UpdateTime) AS LatestPublish - FROM {view_name} - WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) - GROUP BY Site, SubmitHost; - """ - elif view_name == 'VJobRecords': - sql_query = f""" - SELECT - Site, - SubmitHost, - ServiceLevelType, - ServiceLevel, - max(UpdateTime) AS LatestPublish - FROM {view_name} - WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) - GROUP BY Site, SubmitHost; - """ - elif view_name == 'VNormalisedSummaries': - sql_query = f""" - SELECT - Site, - SubmitHost, - ServiceLevelType, - (NormalisedWallDuration / WallDuration) AS ServiceLevel, - max(UpdateTime) AS LatestPublish - FROM {view_name} - WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) - AND WallDuration > 0 - GROUP BY Site, SubmitHost; - """ - else: - log.warning(f"Unknown view name: {view_name}") - return - - # Dynamically get the model class from globals - model_class = globals()[view_name] - fetchset = model_class.objects.using('grid').raw(sql_query) - - for f in fetchset: - BenchmarksBySubmithost.objects.update_or_create( - defaults={ - 'UpdateTime': f.LatestPublish, - 'SourceView': view_name - }, - SiteName=f.Site, - SubmitHost=f.SubmitHost, - ServiceLevelType=f.ServiceLevelType, - ServiceLevel=f.ServiceLevel, - ) - - log.info(f"Refreshed BenchmarksBySubmitHost from {view_name}") - - except Exception: - log.exception(f'Error while trying to refresh BenchmarksBySubmitHost from {view_name}') + log.exception(f'Error while trying to refresh BenchmarksBySubmitHost from {view_name}') if __name__ == "__main__": @@ -390,4 +314,4 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): "Data retrieval and processing attempted. " "Check the above logs for details on the sync status" ) - log.info('=====================') + log.info('=====================') \ No newline at end of file From 34886bd6e3d17be945e4c1d484213ee32fe1afa9 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 19 Aug 2025 09:21:45 +0100 Subject: [PATCH 153/183] Import models --- monitoring/db_update_sqlite.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 4333249..caf7ce9 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -46,7 +46,12 @@ get_year_month_str ) -from monitoring.benchmarks.models import BenchmarksBySubmithost +from monitoring.benchmarks.models import ( + BenchmarksBySubmithost, + VJobRecords, + VSummaries, + VNormalisedSummaries, +) try: # Read configuration from the file From 9e99240b0af81e6db9e46df77d53b4fc44aa15dc Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 19 Aug 2025 10:55:31 +0100 Subject: [PATCH 154/183] Update header --- monitoring/benchmarks/templates/benchmarks_by_submithost.html | 2 +- monitoring/benchmarks/views.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/monitoring/benchmarks/templates/benchmarks_by_submithost.html b/monitoring/benchmarks/templates/benchmarks_by_submithost.html index e8a3c96..882b206 100644 --- a/monitoring/benchmarks/templates/benchmarks_by_submithost.html +++ b/monitoring/benchmarks/templates/benchmarks_by_submithost.html @@ -7,7 +7,7 @@ -

    Sites publishing benchmark records in last 2 months

    +

    Sites publishing benchmark records in last 3 months

    Page last updated: {{ last_fetched|date:"Y-m-d H:i:s.u" }}

    diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py index 8b401b4..0725898 100644 --- a/monitoring/benchmarks/views.py +++ b/monitoring/benchmarks/views.py @@ -15,7 +15,6 @@ class BenchmarksViewSet(viewsets.ReadOnlyModelViewSet): queryset = BenchmarksBySubmithost.objects.all() serializer_class = BenchmarksBySubmithostSerializer template_name = 'benchmarks_by_submithost.html' - lookup_field = 'SiteName' def list(self, request): last_fetched = BenchmarksBySubmithost.objects.aggregate(Max('fetched'))['fetched__max'] From 3173166a4ac655552e92f5985c812664a1ba46bc Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Tue, 19 Aug 2025 13:22:53 +0100 Subject: [PATCH 155/183] Rename columns and add new line --- monitoring/benchmarks/models.py | 4 ++++ .../benchmarks/templates/benchmarks_by_submithost.html | 6 +++--- monitoring/benchmarks/urls.py | 2 +- monitoring/db_update_sqlite.py | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/monitoring/benchmarks/models.py b/monitoring/benchmarks/models.py index afc51e1..47e2ca0 100644 --- a/monitoring/benchmarks/models.py +++ b/monitoring/benchmarks/models.py @@ -1,5 +1,6 @@ from django.db import models + class BenchmarksBySubmithost(models.Model): fetched = models.DateTimeField(auto_now=True) SiteName = models.CharField(max_length=255) @@ -12,6 +13,7 @@ class BenchmarksBySubmithost(models.Model): class Meta: ordering = ('SiteName',) + class VJobRecords(models.Model): Site = models.CharField(max_length=255, primary_key=True) SubmitHost = models.CharField(max_length=255) @@ -25,6 +27,7 @@ class Meta: db_table = 'VJobRecords' verbose_name = 'Job Record' + class VSummaries(models.Model): Site = models.CharField(max_length=255, primary_key=True) SubmitHost = models.CharField(max_length=255) @@ -37,6 +40,7 @@ class Meta: db_table = 'VSummaries' verbose_name = 'Summary' + class VNormalisedSummaries(models.Model): Site = models.CharField(max_length=255, primary_key=True) SubmitHost = models.CharField(max_length=255) diff --git a/monitoring/benchmarks/templates/benchmarks_by_submithost.html b/monitoring/benchmarks/templates/benchmarks_by_submithost.html index 882b206..8556830 100644 --- a/monitoring/benchmarks/templates/benchmarks_by_submithost.html +++ b/monitoring/benchmarks/templates/benchmarks_by_submithost.html @@ -13,9 +13,9 @@

    Sites publishing benchmark records in last 3 months

     - - - + + + {% for benchmark in benchmarks %} diff --git a/monitoring/benchmarks/urls.py b/monitoring/benchmarks/urls.py index 8f3b790..a03135d 100644 --- a/monitoring/benchmarks/urls.py +++ b/monitoring/benchmarks/urls.py @@ -8,4 +8,4 @@ urlpatterns = [ path('', include(router.urls)), -] \ No newline at end of file +] diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index caf7ce9..6b25070 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -319,4 +319,4 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): "Data retrieval and processing attempted. " "Check the above logs for details on the sync status" ) - log.info('=====================') \ No newline at end of file + log.info('=====================') From 3671d89ccd3940d25c26e59516719ef0cb26df92 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Wed, 20 Aug 2025 09:36:51 +0100 Subject: [PATCH 156/183] Trim trailing whitespace --- monitoring/benchmarks/models.py | 2 +- monitoring/db_update_sqlite.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/benchmarks/models.py b/monitoring/benchmarks/models.py index 47e2ca0..2b59d49 100644 --- a/monitoring/benchmarks/models.py +++ b/monitoring/benchmarks/models.py @@ -51,4 +51,4 @@ class VNormalisedSummaries(models.Model): class Meta: managed = False db_table = 'VNormalisedSummaries' - verbose_name = 'Normalised Summary' + verbose_name = 'Normalised Summary' diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 6b25070..c0166d9 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -304,7 +304,7 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): log.info(f"Refreshed BenchmarksBySubmitHost from {view_name}") except Exception: - log.exception(f'Error while trying to refresh BenchmarksBySubmitHost from {view_name}') + log.exception(f'Error while trying to refresh BenchmarksBySubmitHost from {view_name}') if __name__ == "__main__": From 5a00011c5402c17528e7ba572b803ccf3b3d6b4b Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Wed, 20 Aug 2025 11:20:04 +0100 Subject: [PATCH 157/183] Make the site sorting case insensitive --- monitoring/benchmarks/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py index 0725898..677bb93 100644 --- a/monitoring/benchmarks/views.py +++ b/monitoring/benchmarks/views.py @@ -2,6 +2,7 @@ from datetime import datetime, timedelta from django.db.models import Max +from django.db.models.functions import Lower from rest_framework import viewsets from rest_framework.renderers import TemplateHTMLRenderer @@ -12,7 +13,8 @@ from monitoring.benchmarks.serializers import BenchmarksBySubmithostSerializer class BenchmarksViewSet(viewsets.ReadOnlyModelViewSet): - queryset = BenchmarksBySubmithost.objects.all() + queryset = BenchmarksBySubmithost.objects.all().annotate(site_name_lower=Lower('SiteName')).order_by('site_name_lower') + serializer_class = BenchmarksBySubmithostSerializer template_name = 'benchmarks_by_submithost.html' From e7ddcf48ddcce798abcfbf4a202de2cd058f3724 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Thu, 21 Aug 2025 10:28:21 +0100 Subject: [PATCH 158/183] Rename columns --- monitoring/benchmarks/models.py | 6 +++--- monitoring/benchmarks/serializers.py | 6 +++--- .../benchmarks/templates/benchmarks_by_submithost.html | 6 +++--- monitoring/db_update_sqlite.py | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/monitoring/benchmarks/models.py b/monitoring/benchmarks/models.py index 2b59d49..72f9f64 100644 --- a/monitoring/benchmarks/models.py +++ b/monitoring/benchmarks/models.py @@ -5,9 +5,9 @@ class BenchmarksBySubmithost(models.Model): fetched = models.DateTimeField(auto_now=True) SiteName = models.CharField(max_length=255) SubmitHost = models.CharField(max_length=255) - ServiceLevelType = models.CharField(max_length=50) - ServiceLevel = models.DecimalField(max_digits=10, decimal_places=3) - SourceView = models.CharField(max_length=50) + BenchmarkType = models.CharField(max_length=50) + BenchmarkValue = models.DecimalField(max_digits=10, decimal_places=3) + RecordType = models.CharField(max_length=50) UpdateTime = models.DateTimeField() class Meta: diff --git a/monitoring/benchmarks/serializers.py b/monitoring/benchmarks/serializers.py index 0c66725..e7eb7cc 100644 --- a/monitoring/benchmarks/serializers.py +++ b/monitoring/benchmarks/serializers.py @@ -14,8 +14,8 @@ class Meta: fields = ( 'SiteName', 'SubmitHost', - 'ServiceLevelType', - 'ServiceLevel', - 'SourceView', + 'BenchmarkType', + 'BenchmarkValue', + 'RecordType', 'UpdateTime', ) diff --git a/monitoring/benchmarks/templates/benchmarks_by_submithost.html b/monitoring/benchmarks/templates/benchmarks_by_submithost.html index 8556830..c87a068 100644 --- a/monitoring/benchmarks/templates/benchmarks_by_submithost.html +++ b/monitoring/benchmarks/templates/benchmarks_by_submithost.html @@ -22,9 +22,9 @@

    Sites publishing benchmark records in last 3 months

     - - - + + + {% endfor %} diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index c0166d9..2d15ca5 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -293,12 +293,12 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): BenchmarksBySubmithost.objects.update_or_create( defaults={ 'UpdateTime': make_aware(f.LatestPublish) if is_naive(f.LatestPublish) else f.LatestPublish, - 'SourceView': model_class._meta.verbose_name + 'RecordType': model_class._meta.verbose_name }, SiteName=f.Site, SubmitHost=f.SubmitHost, - ServiceLevelType=f.ServiceLevelType, - ServiceLevel=f.ServiceLevel, + BenchmarkType=f.ServiceLevelType, + BenchmarkValue=f.ServiceLevel, ) log.info(f"Refreshed BenchmarksBySubmitHost from {view_name}") From 41440a21f37985e47945b1177b0b8ed319d31565 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Thu, 21 Aug 2025 11:22:57 +0100 Subject: [PATCH 159/183] Remove annotate --- monitoring/benchmarks/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py index 677bb93..866cfad 100644 --- a/monitoring/benchmarks/views.py +++ b/monitoring/benchmarks/views.py @@ -13,7 +13,7 @@ from monitoring.benchmarks.serializers import BenchmarksBySubmithostSerializer class BenchmarksViewSet(viewsets.ReadOnlyModelViewSet): - queryset = BenchmarksBySubmithost.objects.all().annotate(site_name_lower=Lower('SiteName')).order_by('site_name_lower') + queryset = BenchmarksBySubmithost.objects.all().order_by(Lower('SiteName')) serializer_class = BenchmarksBySubmithostSerializer template_name = 'benchmarks_by_submithost.html' From 18c209d39330bda3400c2cad6b1fe9d5521f6780 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 15:11:08 +0100 Subject: [PATCH 160/183] Remove trailing whitespace in update script --- monitoring/db_update_sqlite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py index 2d15ca5..2fc2a77 100644 --- a/monitoring/db_update_sqlite.py +++ b/monitoring/db_update_sqlite.py @@ -51,7 +51,7 @@ VJobRecords, VSummaries, VNormalisedSummaries, -) +) try: # Read configuration from the file @@ -242,7 +242,7 @@ def refresh_BenchmarksBySubmitHost(): def refresh_BenchmarksBySubmitHost_from_view(view_name): - try: + try: if view_name == 'VSummaries': sql_query = f""" SELECT @@ -264,7 +264,7 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name): ServiceLevel, max(UpdateTime) AS LatestPublish FROM {view_name} - WHERE EndTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + WHERE EndTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) AND UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) GROUP BY Site, SubmitHost; """ From af41d31dc8e13bc17ee7538a474837934c7a8f4e Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Thu, 21 Aug 2025 13:25:27 +0100 Subject: [PATCH 161/183] Add site counts by record type --- .../templates/benchmarks_by_submithost.html | 13 +++++++++++++ monitoring/benchmarks/views.py | 13 +++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/monitoring/benchmarks/templates/benchmarks_by_submithost.html b/monitoring/benchmarks/templates/benchmarks_by_submithost.html index c87a068..9b8d0d7 100644 --- a/monitoring/benchmarks/templates/benchmarks_by_submithost.html +++ b/monitoring/benchmarks/templates/benchmarks_by_submithost.html @@ -9,6 +9,19 @@

    Sites publishing benchmark records in last 3 months

    Page last updated: {{ last_fetched|date:"Y-m-d H:i:s.u" }}

    +
    Site Submit hostService level typeService levelSource viewBenchmark typeBenchmark valueRecord type Last updated
    {{ benchmark.SiteName }} {{ benchmark.SubmitHost }}{{ benchmark.ServiceLevelType }}{{ benchmark.ServiceLevel }}{{ benchmark.SourceView }}{{ benchmark.BenchmarkType }}{{ benchmark.BenchmarkValue }}{{ benchmark.RecordType }} {{ benchmark.UpdateTime|date:"Y-m-d H:i:s" }}
    + + + + + {% for item in site_counts_by_record_type %} + + + + + {% endfor %} +
    Record typeNumber of Sites
    {{ item.RecordType }}{{ item.site_count }}
    + diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py index 866cfad..d45ab66 100644 --- a/monitoring/benchmarks/views.py +++ b/monitoring/benchmarks/views.py @@ -1,7 +1,7 @@ from django.shortcuts import render from datetime import datetime, timedelta -from django.db.models import Max +from django.db.models import Max, Count from django.db.models.functions import Lower from rest_framework import viewsets @@ -25,10 +25,19 @@ def list(self, request): response = super(BenchmarksViewSet, self).list(request) + # Count number of distinct sites per RecordType + site_counts_by_record_type = ( + BenchmarksBySubmithost.objects + .values('RecordType') + .annotate(site_count=Count('SiteName', distinct=True)) + .order_by('RecordType') + ) + if type(request.accepted_renderer) is TemplateHTMLRenderer: response.data = { 'benchmarks': response.data, - 'last_fetched': last_fetched + 'last_fetched': last_fetched, + 'site_counts_by_record_type': site_counts_by_record_type } return response From 8f8efa64e8e677ac340eb53526fe90f3838d7f35 Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Thu, 21 Aug 2025 15:18:04 +0100 Subject: [PATCH 162/183] Add new line --- monitoring/benchmarks/templates/benchmarks_by_submithost.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monitoring/benchmarks/templates/benchmarks_by_submithost.html b/monitoring/benchmarks/templates/benchmarks_by_submithost.html index 9b8d0d7..c421801 100644 --- a/monitoring/benchmarks/templates/benchmarks_by_submithost.html +++ b/monitoring/benchmarks/templates/benchmarks_by_submithost.html @@ -22,6 +22,8 @@

    Sites publishing benchmark records in last 3 months

    {% endfor %}
    Site
    +
    + From 159240893abfc3e8bcfaee6eaa8c1ab5ae15b1ac Mon Sep 17 00:00:00 2001 From: Manoj Garai Date: Thu, 21 Aug 2025 16:10:42 +0100 Subject: [PATCH 163/183] Trim trailing whitespace --- monitoring/benchmarks/templates/benchmarks_by_submithost.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/benchmarks/templates/benchmarks_by_submithost.html b/monitoring/benchmarks/templates/benchmarks_by_submithost.html index c421801..f2e3f81 100644 --- a/monitoring/benchmarks/templates/benchmarks_by_submithost.html +++ b/monitoring/benchmarks/templates/benchmarks_by_submithost.html @@ -22,7 +22,7 @@

    Sites publishing benchmark records in last 3 months

    {% endfor %}
    Site
    -
    +
    From d8c31145798df5d9bd13e210b1b5b025d7f15317 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 11:21:13 +0100 Subject: [PATCH 164/183] Improve HTML template for cloud page - Correct info line on how far back the data goes. - Set last update time to be in ISO 8601 format. - Change "page last updated" to "data last fetched" as it's more accurate. --- monitoring/publishing/templates/cloudsites.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/templates/cloudsites.html b/monitoring/publishing/templates/cloudsites.html index 871ad9d..368c3f9 100644 --- a/monitoring/publishing/templates/cloudsites.html +++ b/monitoring/publishing/templates/cloudsites.html @@ -8,8 +8,8 @@ -

    Sites publishing cloud accounting records in last 3 years

    -

    Page last updated: {{ last_fetched|date:"Y-m-d H:i:s.u" }}

    +

    Sites publishing cloud accounting records in the last year

    +

    Data last fetched: {{ last_fetched|date:"c" }}

    From 502ba90235909e222ba18197970aff0359ef5213 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 12:23:12 +0100 Subject: [PATCH 165/183] Tidy HTML and improve text - Standarise indents to two spaces. - Remove extra html tag. - Add closing html tag. - Tweak column headers. - Tidy formatting. - Standardise last fetch timestampts to ISO-8601. --- .../publishing/templates/gridsites.html | 35 ++++++++--------- monitoring/publishing/templates/gridsync.html | 32 ++++++++-------- .../templates/gridsync_singlesite.html | 32 ++++++++-------- .../templates/gridsync_submithost.html | 38 +++++++++---------- 4 files changed, 71 insertions(+), 66 deletions(-) diff --git a/monitoring/publishing/templates/gridsites.html b/monitoring/publishing/templates/gridsites.html index f2330a9..7b742e3 100644 --- a/monitoring/publishing/templates/gridsites.html +++ b/monitoring/publishing/templates/gridsites.html @@ -1,29 +1,30 @@ {% load static %} - - - - -APEL Publication Summary - - - + + + + APEL Publication Summary + + +

    APEL Publication Test

      -
    • Displays the last time the site published accounting data to the GOC. -
    • A warning / error is raised if the site has not published accounting data for 7 / 31 days, if a site has not published data for 31 days, which usually signifies a problem with APEL or RGMA services. -
    • Information about APEL APEL Wiki -
    • Contact: apel-admins [at] stfc.ac.uk -
    • lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}
    +
  • Displays the last time the site published accounting data to APEL. +
  • A warning / error is raised if the site has not published accounting data for 7 / 31 days, if a site has not published data for 31 days. +
  • Information about APEL: EGI Documentation +
  • Data last fetched: {{ last_fetched|date:"c"|slice:":22" }} + + +
    • SiteVMsInLastUpdateCloudTypeLastUpdated
    - - - - + + + + {% for site in sites %} diff --git a/monitoring/publishing/templates/gridsync.html b/monitoring/publishing/templates/gridsync.html index c663b1c..af503be 100644 --- a/monitoring/publishing/templates/gridsync.html +++ b/monitoring/publishing/templates/gridsync.html @@ -1,21 +1,23 @@ {% load static %} - - - - APEL Publication Summary - - - + + + + APEL Publication Summary + + +

    APEL Synchronisation Test

      -
    • A comparison is made between your local APEL database, and the data that you have published to the GOC. -
    • Major differences are flagged with FAIL. -
    • Information about APEL APEL Wiki -
    • Contact: apel-admins [at] stfc.ac.uk -
    • lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}
    +
  • A comparison is made between your local accounting, and the data that you have published to APEL. +
  • Major differences are flagged with FAIL. +
  • Information about APEL: EGI Documentation +
  • Data last fetched: {{ last_fetched|date:"c"|slice:":22" }} + + +
    • ExecutingSiteMeasurementDateMeasurementTimePublication
    Status    		SiteMeasurement DateMeasurement TimePublication Status
    @@ -37,9 +39,9 @@

    APEL Synchronisation Test

    - - + {% endfor %} -
    All sites
    {{ record.RecordEnd }} {{ record.RecordCountPublished }} {{ record.RecordCountInDb }}{{ record.SyncStatus }}{{ record.SyncStatus }}
    + + diff --git a/monitoring/publishing/templates/gridsync_singlesite.html b/monitoring/publishing/templates/gridsync_singlesite.html index 169a043..fd0b7ec 100644 --- a/monitoring/publishing/templates/gridsync_singlesite.html +++ b/monitoring/publishing/templates/gridsync_singlesite.html @@ -1,21 +1,23 @@ {% load static %} - - - - APEL Publication Summary - - - + + + + APEL Publication Summary + + +

    APEL Synchronisation Test

      -
    • A comparison is made between your local APEL database, and the data that you have published to the GOC. -
    • Major differences are flagged with FAIL. -
    • Information about APEL APEL Wiki -
    • Contact: apel-admins [at] stfc.ac.uk -
    • lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}
    +
  • A comparison is made between your local accounting, and the data that you have published to APEL. +
  • Major differences are flagged with FAIL. +
  • Information about APEL: EGI Documentation +
  • Data last fetched: {{ last_fetched|date:"c"|slice:":22" }} + + +
    @@ -35,9 +37,9 @@

    APEL Synchronisation Test

    - - + {% endfor %} -
    {{ records.0.SiteName }}
    {{ record.RecordEnd }} {{ record.RecordCountPublished }} {{ record.RecordCountInDb }}{{ record.SyncStatus }}{{ record.SyncStatus }}
    + + diff --git a/monitoring/publishing/templates/gridsync_submithost.html b/monitoring/publishing/templates/gridsync_submithost.html index 6bb115e..6ed0cd7 100644 --- a/monitoring/publishing/templates/gridsync_submithost.html +++ b/monitoring/publishing/templates/gridsync_submithost.html @@ -1,45 +1,45 @@ {% load static %} - - - - APEL Publication Summary - - - + + + + APEL Publication Summary + + +

    APEL Synchronisation Test

      -
    • A comparison is made between your local APEL database, and the data that you have published to the GOC. -
    • Major differences are flagged with FAIL. -
    • Information about APEL APEL Wiki -
    • Contact: apel-admins [at] stfc.ac.uk -
    • lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}
    +
  • A comparison is made between your local accounting, and the data that you have published to APEL. +
  • Major differences are flagged with FAIL. +
  • Information about APEL: EGI Documentation +
  • Data last fetched: {{ last_fetched|date:"c"|slice:":22" }} + + +
    - + - - + + {% for host in submisthosts %} - - - {% endfor %} -
    {{submisthosts.0.SiteName}}, {{submisthosts.0.YearMonth}}
    {{ submisthosts.0.SiteName }}, {{ submisthosts.0.YearMonth }}
    Month SubmitHostRecordStartRecordEndRecord StartRecord End Record Count
    In Your Database    		 Record Count
    What You Published
    {{ host.YearMonth }} {{ host.SubmitHost }} {{ host.RecordStart }} {{ host.RecordEnd }} {{ host.RecordCountPublished }} {{ host.RecordCountInDb }}
    + + From f50e8edf21f903ac432169433d25459568db5ab9 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 12:31:10 +0100 Subject: [PATCH 166/183] Add missing semi-colon to stylesheet --- monitoring/publishing/static/stylesheet.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/publishing/static/stylesheet.css b/monitoring/publishing/static/stylesheet.css index 88f63af..ad8e70a 100644 --- a/monitoring/publishing/static/stylesheet.css +++ b/monitoring/publishing/static/stylesheet.css @@ -32,7 +32,7 @@ th { font-size: 12px; color: #009999; background-color: #FFFFFF; - font-weight: bold + font-weight: bold; align: left; } a:link { From 84fe8765eae7b59bf77adf2cf56745dbc77008d6 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 12:37:12 +0100 Subject: [PATCH 167/183] Fix cloud page tag(s) --- monitoring/publishing/templates/cloudsites.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/monitoring/publishing/templates/cloudsites.html b/monitoring/publishing/templates/cloudsites.html index 368c3f9..0ffc1ef 100644 --- a/monitoring/publishing/templates/cloudsites.html +++ b/monitoring/publishing/templates/cloudsites.html @@ -1,9 +1,7 @@ - - - + Sites publishing cloud accounting records From bbe9f1ab55c4a94a7635ed8b7b87a51bbbca8b71 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 12:42:27 +0100 Subject: [PATCH 168/183] Fix misplaced link tag --- monitoring/publishing/templates/gridsync.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/publishing/templates/gridsync.html b/monitoring/publishing/templates/gridsync.html index af503be..6ca4d06 100644 --- a/monitoring/publishing/templates/gridsync.html +++ b/monitoring/publishing/templates/gridsync.html @@ -33,8 +33,8 @@

    APEL Synchronisation Test

    {% for record in records.results %} - {{ record.SiteName }} - {{ record.YearMonth }} + {{ record.SiteName }} + {{ record.YearMonth }} {{ record.RecordStart }} {{ record.RecordEnd }} {{ record.RecordCountPublished }} From 4e1c338d602bcf496e85c0881b46ab65810f3ef3 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 13:52:13 +0100 Subject: [PATCH 169/183] Increase font size to match other text --- monitoring/publishing/static/stylesheet.css | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/monitoring/publishing/static/stylesheet.css b/monitoring/publishing/static/stylesheet.css index ad8e70a..11ce5f0 100644 --- a/monitoring/publishing/static/stylesheet.css +++ b/monitoring/publishing/static/stylesheet.css @@ -65,7 +65,7 @@ hr { } .navbar-title { font-family: Arial, Helvetica, sans-serif; - font-size: 0.8em; + font-size: 1em; color: #FFFFFF; font-weight: bold; @@ -122,37 +122,37 @@ li { } .tabletext { font-family: Arial, Helvetica, sans-serif; - font-size: 0.8em; + font-size: 1em; color: #000000; background-color: #DDDDDD; } .tabletextwarning { font-family: Arial, Helvetica, sans-serif; - font-size: 0.8em; + font-size: 1em; color: #000000; background-color: #FFFF00; } .tabletextok { font-family: Arial, Helvetica, sans-serif; - font-size: 0.8em; + font-size: 1em; color: #000000; background-color: #00FF00; } .tabletexterror { font-family: Arial, Helvetica, sans-serif; - font-size: 0.8em; + font-size: 1em; color: #000000; background-color: #FF0000; } .tabletextinfo { font-family: Arial, Helvetica, sans-serif; - font-size: 0.8em; + font-size: 1em; color: #000000; background-color: #00CCFF; } .tableheader { font-family: Arial, Helvetica, sans-serif; - font-size: 0.8em; + font-size: 1em; color: #FFFFFF; background-color: #000000; font-weight: bold; @@ -160,7 +160,7 @@ li { } .navbar-heading { font-family: Arial, Helvetica, sans-serif; - font-size: 0.8em; + font-size: 1em; background-color: #FFFFFF; font-weight: bold; color: #000000; From 84668d29f9b8981aeb3da3f3139ff09af40b0441 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 13:52:38 +0100 Subject: [PATCH 170/183] Add missing style to table contents --- monitoring/publishing/templates/gridsync.html | 16 ++++++++-------- .../templates/gridsync_singlesite.html | 16 +++++++++------- .../templates/gridsync_submithost.html | 12 ++++++------ 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/monitoring/publishing/templates/gridsync.html b/monitoring/publishing/templates/gridsync.html index 6ca4d06..1713bd7 100644 --- a/monitoring/publishing/templates/gridsync.html +++ b/monitoring/publishing/templates/gridsync.html @@ -28,18 +28,18 @@

    APEL Synchronisation Test

    Record End Record Count
    In Your Database Record Count
    What You Published - Synchronisation
    Status + Synchronisation Status {% for record in records.results %} - {{ record.SiteName }} - {{ record.YearMonth }} - {{ record.RecordStart }} - {{ record.RecordEnd }} - {{ record.RecordCountPublished }} - {{ record.RecordCountInDb }} - {{ record.SyncStatus }} + {{ record.SiteName }} + {{ record.YearMonth }} + {{ record.RecordStart }} + {{ record.RecordEnd }} + {{ record.RecordCountPublished }} + {{ record.RecordCountInDb }} + {{ record.SyncStatus }} {% endfor %} diff --git a/monitoring/publishing/templates/gridsync_singlesite.html b/monitoring/publishing/templates/gridsync_singlesite.html index fd0b7ec..c9d37b3 100644 --- a/monitoring/publishing/templates/gridsync_singlesite.html +++ b/monitoring/publishing/templates/gridsync_singlesite.html @@ -27,17 +27,19 @@

    APEL Synchronisation Test

    Record End Record Count
    In Your Database Record Count
    What You Published - Synchronisation
    Status + Synchronisation Status {% for record in records %} - {{ record.YearMonth }} - {{ record.RecordStart }} - {{ record.RecordEnd }} - {{ record.RecordCountPublished }} - {{ record.RecordCountInDb }} - {{ record.SyncStatus }} + + {{ record.YearMonth }} + + {{ record.RecordStart }} + {{ record.RecordEnd }} + {{ record.RecordCountPublished }} + {{ record.RecordCountInDb }} + {{ record.SyncStatus }} {% endfor %} diff --git a/monitoring/publishing/templates/gridsync_submithost.html b/monitoring/publishing/templates/gridsync_submithost.html index 6ed0cd7..cedb8df 100644 --- a/monitoring/publishing/templates/gridsync_submithost.html +++ b/monitoring/publishing/templates/gridsync_submithost.html @@ -32,12 +32,12 @@

    APEL Synchronisation Test

    {% for host in submisthosts %} - {{ host.YearMonth }} - {{ host.SubmitHost }} - {{ host.RecordStart }} - {{ host.RecordEnd }} - {{ host.RecordCountPublished }} - {{ host.RecordCountInDb }} + {{ host.YearMonth }} + {{ host.SubmitHost }} + {{ host.RecordStart }} + {{ host.RecordEnd }} + {{ host.RecordCountPublished }} + {{ host.RecordCountInDb }} {% endfor %} From 370657ff8827178b38adeaf384fe79a43125732c Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 14:16:45 +0100 Subject: [PATCH 171/183] Add humanised formatting to the large numbers --- monitoring/publishing/templates/gridsync.html | 5 +++-- monitoring/publishing/templates/gridsync_singlesite.html | 5 +++-- monitoring/publishing/templates/gridsync_submithost.html | 5 +++-- monitoring/settings.py | 1 + 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/monitoring/publishing/templates/gridsync.html b/monitoring/publishing/templates/gridsync.html index 1713bd7..8c49d4e 100644 --- a/monitoring/publishing/templates/gridsync.html +++ b/monitoring/publishing/templates/gridsync.html @@ -1,4 +1,5 @@ {% load static %} +{% load humanize %} @@ -37,8 +38,8 @@

    APEL Synchronisation Test

    {{ record.YearMonth }} {{ record.RecordStart }} {{ record.RecordEnd }} - {{ record.RecordCountPublished }} - {{ record.RecordCountInDb }} + {{ record.RecordCountPublished|intcomma }} + {{ record.RecordCountInDb|intcomma }} {{ record.SyncStatus }} {% endfor %} diff --git a/monitoring/publishing/templates/gridsync_singlesite.html b/monitoring/publishing/templates/gridsync_singlesite.html index c9d37b3..d332194 100644 --- a/monitoring/publishing/templates/gridsync_singlesite.html +++ b/monitoring/publishing/templates/gridsync_singlesite.html @@ -1,4 +1,5 @@ {% load static %} +{% load humanize %} @@ -37,8 +38,8 @@

    APEL Synchronisation Test

    {{ record.RecordStart }} {{ record.RecordEnd }} - {{ record.RecordCountPublished }} - {{ record.RecordCountInDb }} + {{ record.RecordCountPublished|intcomma }} + {{ record.RecordCountInDb|intcomma }} {{ record.SyncStatus }} {% endfor %} diff --git a/monitoring/publishing/templates/gridsync_submithost.html b/monitoring/publishing/templates/gridsync_submithost.html index cedb8df..28f1dbb 100644 --- a/monitoring/publishing/templates/gridsync_submithost.html +++ b/monitoring/publishing/templates/gridsync_submithost.html @@ -1,4 +1,5 @@ {% load static %} +{% load humanize %} @@ -36,8 +37,8 @@

    APEL Synchronisation Test

    {{ host.SubmitHost }} {{ host.RecordStart }} {{ host.RecordEnd }} - {{ host.RecordCountPublished }} - {{ host.RecordCountInDb }} + {{ host.RecordCountPublished|intcomma }} + {{ host.RecordCountInDb|intcomma }} {% endfor %} diff --git a/monitoring/settings.py b/monitoring/settings.py index e00bb9d..8f7d710 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -70,6 +70,7 @@ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', + 'django.contrib.humanize', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', From 48b8a4b050d6deb8d50cbbefef03abf81150ad46 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 15:36:18 +0100 Subject: [PATCH 172/183] Update site base urls to use newer path() function --- monitoring/urls.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/monitoring/urls.py b/monitoring/urls.py index 7d08c4f..15801ce 100644 --- a/monitoring/urls.py +++ b/monitoring/urls.py @@ -1,26 +1,10 @@ -"""Monitoring URL Configuration. - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/1.11/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.conf.urls import url, include - 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) -""" -from django.conf.urls import include, url +from django.urls import include, path from django.contrib import admin urlpatterns = [ - url(r'^admin/', admin.site.urls), - url(r'^availability/', include('monitoring.availability.urls')), - url(r'^publishing/', include('monitoring.publishing.urls')), - url(r'^benchmarks/', include('monitoring.benchmarks.urls')), - url(r'^api-auth/', include('rest_framework.urls', - namespace='rest_framework')), + path('admin/', admin.site.urls), + path('availability/', include('monitoring.availability.urls')), + path('publishing/', include('monitoring.publishing.urls')), + path('benchmarks/', include('monitoring.benchmarks.urls')), + path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), ] From 521253126f5a58a19a822a4980792edb6244926c Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 16:00:58 +0100 Subject: [PATCH 173/183] Add basic home page --- monitoring/settings.py | 2 +- monitoring/templates/home.html | 9 +++++++++ monitoring/urls.py | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 monitoring/templates/home.html diff --git a/monitoring/settings.py b/monitoring/settings.py index 8f7d710..da4991a 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -103,7 +103,7 @@ TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], + 'DIRS': ['templates',], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ diff --git a/monitoring/templates/home.html b/monitoring/templates/home.html new file mode 100644 index 0000000..39585d6 --- /dev/null +++ b/monitoring/templates/home.html @@ -0,0 +1,9 @@ + + + + +

    APEL Data Validation

    +

    Welcome. This page gives an overview of the views available in the APEL Data Validation system.

    + + + diff --git a/monitoring/urls.py b/monitoring/urls.py index 15801ce..a2c2509 100644 --- a/monitoring/urls.py +++ b/monitoring/urls.py @@ -1,8 +1,10 @@ from django.urls import include, path from django.contrib import admin +from django.views.generic import TemplateView urlpatterns = [ path('admin/', admin.site.urls), + path('', TemplateView.as_view(template_name='home.html')), path('availability/', include('monitoring.availability.urls')), path('publishing/', include('monitoring.publishing.urls')), path('benchmarks/', include('monitoring.benchmarks.urls')), From d085961825a3e34b24ca1a7aec8ad3c469a1a205 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 17:23:28 +0100 Subject: [PATCH 174/183] Fix template dir path --- monitoring/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monitoring/settings.py b/monitoring/settings.py index da4991a..65d0352 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -103,7 +103,8 @@ TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': ['templates',], + # Add project-wide templates directory + 'DIRS': [os.path.join(BASE_DIR, 'monitoring', 'templates'),], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ From 27a0437aeffd02a20a97c5e79a53b7856fb447e5 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 17:57:56 +0100 Subject: [PATCH 175/183] Add names to project urls for later referencing --- monitoring/urls.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monitoring/urls.py b/monitoring/urls.py index a2c2509..ac77b36 100644 --- a/monitoring/urls.py +++ b/monitoring/urls.py @@ -4,9 +4,9 @@ urlpatterns = [ path('admin/', admin.site.urls), - path('', TemplateView.as_view(template_name='home.html')), - path('availability/', include('monitoring.availability.urls')), + path('', TemplateView.as_view(template_name='home.html'), name='home'), + path('availability/', include('monitoring.availability.urls'), name='availability'), path('publishing/', include('monitoring.publishing.urls')), - path('benchmarks/', include('monitoring.benchmarks.urls')), + path('benchmarks/', include('monitoring.benchmarks.urls'), name='benchmarks'), path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), ] From eb21d6af6663ad2ff56091cfbd8ec0e0155ed329 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 17:58:25 +0100 Subject: [PATCH 176/183] Add list of hyperlinked pages to home page --- monitoring/templates/home.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/monitoring/templates/home.html b/monitoring/templates/home.html index 39585d6..cdde682 100644 --- a/monitoring/templates/home.html +++ b/monitoring/templates/home.html @@ -5,5 +5,13 @@

    APEL Data Validation

    Welcome. This page gives an overview of the views available in the APEL Data Validation system.

    + + From 0ed1f1d6bc7f503b85620fde1f40d1fe0aa5e243 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 19:01:42 +0100 Subject: [PATCH 177/183] Reorganise URL naming --- monitoring/availability/urls.py | 2 +- monitoring/benchmarks/urls.py | 5 +---- monitoring/templates/home.html | 8 ++++---- monitoring/urls.py | 4 ++-- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/monitoring/availability/urls.py b/monitoring/availability/urls.py index 3b2d575..15b7627 100644 --- a/monitoring/availability/urls.py +++ b/monitoring/availability/urls.py @@ -3,5 +3,5 @@ from monitoring.availability import views urlpatterns = [ - url(r'^$', views.status), + url(r'^$', views.status, name='availability'), ] diff --git a/monitoring/benchmarks/urls.py b/monitoring/benchmarks/urls.py index a03135d..e3b06cb 100644 --- a/monitoring/benchmarks/urls.py +++ b/monitoring/benchmarks/urls.py @@ -1,11 +1,8 @@ from rest_framework import routers -from django.urls import path, include from monitoring.benchmarks import views router = routers.SimpleRouter() router.register('', views.BenchmarksViewSet) -urlpatterns = [ - path('', include(router.urls)), -] +urlpatterns = router.urls diff --git a/monitoring/templates/home.html b/monitoring/templates/home.html index cdde682..99b1047 100644 --- a/monitoring/templates/home.html +++ b/monitoring/templates/home.html @@ -7,10 +7,10 @@

    APEL Data Validation

    diff --git a/monitoring/urls.py b/monitoring/urls.py index ac77b36..10c97b9 100644 --- a/monitoring/urls.py +++ b/monitoring/urls.py @@ -5,8 +5,8 @@ urlpatterns = [ path('admin/', admin.site.urls), path('', TemplateView.as_view(template_name='home.html'), name='home'), - path('availability/', include('monitoring.availability.urls'), name='availability'), + path('availability/', include('monitoring.availability.urls')), path('publishing/', include('monitoring.publishing.urls')), - path('benchmarks/', include('monitoring.benchmarks.urls'), name='benchmarks'), + path('benchmarks/', include('monitoring.benchmarks.urls')), path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), ] From 5f8bf5096507851eeb910aa63a090f30eed2f3d3 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 21:27:12 +0100 Subject: [PATCH 178/183] Use correct URL name --- monitoring/templates/home.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/templates/home.html b/monitoring/templates/home.html index 99b1047..2141823 100644 --- a/monitoring/templates/home.html +++ b/monitoring/templates/home.html @@ -10,7 +10,7 @@

    APEL Data Validation

  • {% url 'cloudsite-list' %}
  • {% url 'gridsite-list' %}
  • {% url 'gridsitesync-list' %}
    • -
  • {% url 'benchmark-list' %}
    • +
  • {% url 'benchmarksbysubmithost-list' %}
    • From b2917abe95549ffbc03e012da5c7f5e40f714fee Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 21:27:44 +0100 Subject: [PATCH 179/183] Remove availability link as not for human parsing --- monitoring/templates/home.html | 1 - 1 file changed, 1 deletion(-) diff --git a/monitoring/templates/home.html b/monitoring/templates/home.html index 2141823..032956f 100644 --- a/monitoring/templates/home.html +++ b/monitoring/templates/home.html @@ -6,7 +6,6 @@

    APEL Data Validation

    Welcome. This page gives an overview of the views available in the APEL Data Validation system.

      -
    • {% url 'availability' %}
    • {% url 'cloudsite-list' %}
    • {% url 'gridsite-list' %}
    • {% url 'gridsitesync-list' %}
      • From 11246cc3ad32468e3e62069e8760b37580c621e7 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Aug 2025 21:29:25 +0100 Subject: [PATCH 180/183] Add more detail and examples --- monitoring/templates/home.html | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/monitoring/templates/home.html b/monitoring/templates/home.html index 032956f..f2fa27a 100644 --- a/monitoring/templates/home.html +++ b/monitoring/templates/home.html @@ -5,12 +5,28 @@

      APEL Data Validation

      Welcome. This page gives an overview of the views available in the APEL Data Validation system.

      +

      EGI/WLCG Grid Accounting

      +

      EGI Cloud Accounting

      +

      + +

      Return format

      +

      + The views above return HTML by default, but can be set to return JSON by adding ?format=json at the end of the URL. + For example: {% url 'gridsitesync-detail' 'RAL-LCG2' %}?format=json +

      + From 754ef7550914e2f843e9f2d64fe2bd9a3486bf04 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 27 Aug 2025 16:18:19 +0100 Subject: [PATCH 181/183] Remove repitition --- monitoring/publishing/templates/gridsites.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/publishing/templates/gridsites.html b/monitoring/publishing/templates/gridsites.html index 7b742e3..dc5426e 100644 --- a/monitoring/publishing/templates/gridsites.html +++ b/monitoring/publishing/templates/gridsites.html @@ -12,7 +12,7 @@

      APEL Publication Test

      • Displays the last time the site published accounting data to APEL. -
      • A warning / error is raised if the site has not published accounting data for 7 / 31 days, if a site has not published data for 31 days. +
      • A warning / error is raised if the site has not published accounting data for 7 / 31 days.
      • Information about APEL: EGI Documentation
      • Data last fetched: {{ last_fetched|date:"c"|slice:":22" }}
      From 8c97355ee375365040988ca5278313a2c2657f7f Mon Sep 17 00:00:00 2001 From: Adrian Coveney <4836233+tofu-rocketry@users.noreply.github.com> Date: Thu, 28 Aug 2025 11:15:46 +0100 Subject: [PATCH 182/183] Convert remaining url() to path() (#58) url() is deprecating and is directly replaced by re_path(), but the new simpler path() function suits this use case. --- monitoring/availability/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/availability/urls.py b/monitoring/availability/urls.py index 15b7627..077876e 100644 --- a/monitoring/availability/urls.py +++ b/monitoring/availability/urls.py @@ -1,7 +1,7 @@ -from django.conf.urls import url +from django.urls import path from monitoring.availability import views urlpatterns = [ - url(r'^$', views.status, name='availability'), + path('', views.status, name='availability'), ] From 896bb3a73578744974e74133d0b2097fbcfc1421 Mon Sep 17 00:00:00 2001 From: garaimanoj <99975605+garaimanoj@users.noreply.github.com> Date: Wed, 3 Sep 2025 15:54:01 +0100 Subject: [PATCH 183/183] Add sorting by update time (#59) * Add sorting by update time * Add comment about ordering --- monitoring/benchmarks/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py index d45ab66..736d6ba 100644 --- a/monitoring/benchmarks/views.py +++ b/monitoring/benchmarks/views.py @@ -13,7 +13,9 @@ from monitoring.benchmarks.serializers import BenchmarksBySubmithostSerializer class BenchmarksViewSet(viewsets.ReadOnlyModelViewSet): - queryset = BenchmarksBySubmithost.objects.all().order_by(Lower('SiteName')) + # Lower('SiteName'): sorts sites alphabetically, case-insensitively. + # '-UpdateTime': sorts records within each site by UpdateTime in descending order (latest first). + queryset = BenchmarksBySubmithost.objects.all().order_by(Lower('SiteName'), '-UpdateTime') serializer_class = BenchmarksBySubmithostSerializer template_name = 'benchmarks_by_submithost.html'