Skip to content

Commit f650f3f

Browse files
henrikekhenrikek
committed
Fix remote status exchange when writes to to multiple storage methods
1 parent 9279c8b commit f650f3f

6 files changed

Lines changed: 97 additions & 48 deletions

File tree

‎ESSArch_EPP/Storage/libs.py‎

Lines changed: 59 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -529,54 +529,56 @@ def get_write_status(self, ArchiveObject_obj):
529529
ArchiveObject_obj.ObjectIdentifierValue,
530530
IOQueue_obj.id)
531531
self.logger.error(event_info)
532-
else:
533-
timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
534-
storageMedium_obj = IOQueue_obj.storagemedium
535-
storageMedium_obj.storageMediumUsedCapacity = storageMedium_obj.storageMediumUsedCapacity + int(IOQueue_obj.result.get('WriteSize'))
536-
storageMedium_obj.linkingAgentIdentifierValue = self.AgentIdentifierValue
537-
storageMedium_obj.storageMediumDate = timestamp_utc
538-
storageMedium_obj.LocalDBdatetime = timestamp_utc
539-
storageMedium_obj.save(update_fields=['storageMediumUsedCapacity','storageMediumDate','linkingAgentIdentifierValue','LocalDBdatetime'])
540-
if self.ExtDBupdate:
541-
ext_res,ext_errno,ext_why = ESSMSSQL.DB().action('storageMedium','UPD',('storageMediumUsedCapacity',storageMedium_obj.storageMediumUsedCapacity,
542-
'storageMediumDate',storageMedium_obj.storageMediumDate.astimezone(self.tz).replace(tzinfo=None),
543-
'linkingAgentIdentifierValue',storageMedium_obj.linkingAgentIdentifierValue),
544-
('storageMediumID',storageMedium_obj.storageMediumID))
545-
if ext_errno: self.logger.error('Failed to update External DB: ' + str(storageMedium_obj.storageMediumID) + ' error: ' + str(ext_why))
546-
else:
547-
storageMedium_obj.ExtDBdatetime = storageMedium_obj.LocalDBdatetime
548-
storageMedium_obj.save(update_fields=['ExtDBdatetime'])
549-
550-
# Flag to update remote server
551-
target_obj = IOQueue_obj.storagemethodtarget.target
552-
remote_server = target_obj.remote_server.split(',')
553-
if len(remote_server) == 3:
554-
# Update remote server
555-
try:
556-
self._update_remote_archiveobject(remote_server, ArchiveObject_obj)
557-
except DatabasePostRestError as e:
558-
error_flag = 1
559-
msg = 'Failed to update remote DB status for AIP: %s, error: %s' % (
560-
ArchiveObject_obj.ObjectIdentifierValue,
561-
e)
562-
self.logger.error(msg)
563-
error_list.append(msg)
564-
565-
event_info = 'Succeeded to write object: %s to storage target: %s (IOuuid: %s)' % (
566-
ArchiveObject_obj.ObjectIdentifierValue,
567-
IOQueue_obj.storagemethodtarget.name,
568-
IOQueue_obj.id)
569-
self.logger.info(event_info)
570-
ESSPGM.Events().create('1101', '', self.__name__, __version__, '0', event_info, 2, ArchiveObject_obj.ObjectIdentifierValue)
571-
IOQueue_obj.Status = 21
572-
IOQueue_obj.save(update_fields=['Status'])
573532
if len(all_storage_target_objs) < len(self.st_objs_to_check[ArchiveObject_obj]):
574533
object_writes_ok_flag = 0
575534
event_info = 'There are fewer storage entrys in the database (%s) of object: %s than is configured in the archive policy (%s)' % (
576535
len(all_storage_target_objs),
577536
ArchiveObject_obj.ObjectIdentifierValue,
578537
len(self.st_objs_to_check))
579538
self.logger.debug(event_info)
539+
540+
for IOQueue_obj in IOQueue_objs:
541+
if IOQueue_obj.Status == 20 and IOQueue_obj.storagemethodtarget.target in all_storage_target_objs and IOQueue_obj.storage in all_storage_objs:
542+
timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
543+
storageMedium_obj = IOQueue_obj.storagemedium
544+
storageMedium_obj.storageMediumUsedCapacity = storageMedium_obj.storageMediumUsedCapacity + int(IOQueue_obj.result.get('WriteSize'))
545+
storageMedium_obj.linkingAgentIdentifierValue = self.AgentIdentifierValue
546+
storageMedium_obj.storageMediumDate = timestamp_utc
547+
storageMedium_obj.LocalDBdatetime = timestamp_utc
548+
storageMedium_obj.save(update_fields=['storageMediumUsedCapacity','storageMediumDate','linkingAgentIdentifierValue','LocalDBdatetime'])
549+
if self.ExtDBupdate:
550+
ext_res,ext_errno,ext_why = ESSMSSQL.DB().action('storageMedium','UPD',('storageMediumUsedCapacity',storageMedium_obj.storageMediumUsedCapacity,
551+
'storageMediumDate',storageMedium_obj.storageMediumDate.astimezone(self.tz).replace(tzinfo=None),
552+
'linkingAgentIdentifierValue',storageMedium_obj.linkingAgentIdentifierValue),
553+
('storageMediumID',storageMedium_obj.storageMediumID))
554+
if ext_errno: self.logger.error('Failed to update External DB: ' + str(storageMedium_obj.storageMediumID) + ' error: ' + str(ext_why))
555+
else:
556+
storageMedium_obj.ExtDBdatetime = storageMedium_obj.LocalDBdatetime
557+
storageMedium_obj.save(update_fields=['ExtDBdatetime'])
558+
559+
# Flag to update remote server
560+
target_obj = IOQueue_obj.storagemethodtarget.target
561+
remote_server = target_obj.remote_server.split(',')
562+
if len(remote_server) == 3:
563+
# Update remote server
564+
try:
565+
self._update_remote_archiveobject(remote_server, ArchiveObject_obj, object_writes_ok_flag)
566+
except DatabasePostRestError as e:
567+
error_flag = 1
568+
msg = 'Failed to update remote DB status for AIP: %s, error: %s' % (
569+
ArchiveObject_obj.ObjectIdentifierValue,
570+
e)
571+
self.logger.error(msg)
572+
error_list.append(msg)
573+
574+
event_info = 'Succeeded to write object: %s to storage target: %s (IOuuid: %s)' % (
575+
ArchiveObject_obj.ObjectIdentifierValue,
576+
IOQueue_obj.storagemethodtarget.name,
577+
IOQueue_obj.id)
578+
self.logger.info(event_info)
579+
ESSPGM.Events().create('1101', '', self.__name__, __version__, '0', event_info, 2, ArchiveObject_obj.ObjectIdentifierValue)
580+
IOQueue_obj.Status = 21
581+
IOQueue_obj.save(update_fields=['Status'])
580582

581583
if error_flag:
582584
raise ESSArchSMError(error_list)
@@ -662,7 +664,7 @@ def handle_write_status(self, ArchiveObject_obj):
662664
IOQueue_objs.delete()
663665

664666
@retry(stop_max_attempt_number=1440, wait_fixed=60000)
665-
def _update_remote_archiveobject(self, remote_server, ArchiveObject_obj):
667+
def _update_remote_archiveobject(self, remote_server, ArchiveObject_obj, object_writes_ok_flag = 1):
666668
""" Call REST service on remote to update ArchiveObject with nested storage, storageMedium
667669
668670
:param remote_server: example: [https://servername:port, user, password]
@@ -676,9 +678,14 @@ def _update_remote_archiveobject(self, remote_server, ArchiveObject_obj):
676678
requests_session.verify = False
677679
requests_session.auth = (ruser, rpass)
678680
ArchiveObject_obj_data = ArchiveObjectPlusAICPlusStorageNestedReadSerializer(ArchiveObject_obj).data
679-
# Set StatusProcess and StatusActivity
680-
ArchiveObject_obj_data['StatusProcess'] = 1999 # 'Write AIP OK'
681-
ArchiveObject_obj_data['StatusActivity'] = 0 # 'OK'
681+
if object_writes_ok_flag == 1:
682+
# Set StatusProcess and StatusActivity
683+
ArchiveObject_obj_data['StatusProcess'] = 1999 # 'Write AIP OK'
684+
ArchiveObject_obj_data['StatusActivity'] = 0 # 'OK'
685+
else:
686+
# Set StatusProcess and StatusActivity
687+
ArchiveObject_obj_data['StatusProcess'] = 1500 # 'Remote AIP'
688+
ArchiveObject_obj_data['StatusActivity'] = 6 # 'Pending writes'
682689
# Remove local disk storage type 200 from Storage_set
683690
#exclude_targets = [i.id for i in StorageTargets.objects.filter(type=200, remote_server='')]
684691
# Remove all storage that not belong to specific remote server from Storage_set
@@ -714,10 +721,16 @@ def _update_remote_archiveobject(self, remote_server, ArchiveObject_obj):
714721
self.logger.warning(msg)
715722
raise DatabasePostRestError(e)
716723
else:
717-
msg = 'Success to update remote server: %s with status(1999) for object: %s, media: %s' % (
724+
if object_writes_ok_flag == 1:
725+
msg = 'Success to update remote server: %s status "all writes are done" for object: %s, media: %s' % (
718726
base_url,
719727
ArchiveObject_obj.ObjectIdentifierValue,
720728
update_info)
729+
else:
730+
msg = 'Success to update remote server: %s status "all writes are not completed" for object: %s, media: %s' % (
731+
base_url,
732+
ArchiveObject_obj.ObjectIdentifierValue,
733+
update_info)
721734
self.logger.info(msg)
722735

723736
@retry(stop_max_attempt_number=5, wait_fixed=60000)

‎ESSArch_EPP/access/templates/access/detail.html‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,3 +60,19 @@ <H1>{{ label }}</H1>
6060

6161

6262
{% endblock %}
63+
64+
{% block app-extra-script %}
65+
<script>
66+
67+
//Reload script
68+
window.onload = setupRefresh;
69+
70+
function setupRefresh() {
71+
setTimeout("refreshPage();", 30000);
72+
}
73+
function refreshPage() {
74+
window.location = location.href;
75+
}
76+
77+
</script>
78+
{% endblock %}

‎ESSArch_EPP/access/templates/access/list.html‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,3 +48,19 @@ <H1>{{ label }}</H1>
4848
</div>
4949

5050
{% endblock %}
51+
52+
{% block app-extra-script %}
53+
<script>
54+
55+
//Reload script
56+
window.onload = setupRefresh;
57+
58+
function setupRefresh() {
59+
setTimeout("refreshPage();", 30000);
60+
}
61+
function refreshPage() {
62+
window.location = location.href;
63+
}
64+
65+
</script>
66+
{% endblock %}

‎ESSArch_EPP/config/settings.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,8 @@
9494
#LANGUAGE_CODE = 'nn' #Norwegian Nynorsk
9595
#LANGUAGE_CODE = 'nb' #Norwegian Bokmal
9696

97+
SESSION_COOKIE_NAME = 'epp'
98+
9799
SITE_ID = 1
98100

99101
# If you set this to False, Django will make some optimizations so as not

‎ESSArch_EPP/config/settings_dev.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,8 @@
128128
#LANGUAGE_CODE = 'nn' #Norwegian Nynorsk
129129
#LANGUAGE_CODE = 'nb' #Norwegian Bokmal
130130

131+
SESSION_COOKIE_NAME = 'epp'
132+
131133
SITE_ID = 1
132134

133135
# If you set this to False, Django will make some optimizations so as not

‎ESSArch_EPP/extra/install_config.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ def installdefaultpaths(): # default paths
393393
#else:
394394
dct = {
395395
'path_mimetypes_definitionfile':'/ESSArch/config/mime.types',
396-
'path_reception':'/ESSArch/data/eta/reception',
396+
'path_reception':'/ESSArch/data/eta/reception/eft',
397397
'path_gate':'/ESSArch/data/gate',
398398
'path_work':'/ESSArch/data/epp/work',
399399
'path_control':'/ESSArch/data/epp/control',
@@ -693,7 +693,7 @@ def installdefaultArchivePolicy(): # default ArchivePolicy
693693
StorageTarget_obj.name=u'Default policy 1 - SM 1 - Target 1'
694694
StorageTarget_obj.status=1
695695
StorageTarget_obj.storagemethod=StorageMethod.objects.get(name=u'Default policy 1 - SM 1')
696-
StorageTarget_obj.target=StorageTargets.objects.get(name=u'disk1')
696+
StorageTarget_obj.target=StorageTargets.objects.get(name__startswith=u'disk1')
697697
StorageTarget_obj.save()
698698

699699
def installdefaultESSProc(): # default ESSProc

0 commit comments

Comments
 (0)