Events listing, deletion and archival optimizations

api/src/main/java/org/apache/cloudstack/api/command/user/event/ArchiveEventsCmd.java

@@ -31,7 +31,6 @@
 import org.apache.cloudstack.context.CallContext;
 
 import com.cloud.event.Event;
-import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.user.Account;
 
 @APICommand(name = "archiveEvents", description = "Archive one or more events.", responseObject = SuccessResponse.class, entityType = {Event.class},
@@ -82,6 +81,22 @@ public String getType() {
         return type;
     }
 
+    public void setIds(List<Long> ids) {
+        this.ids = ids;
+    }
+
+    public void setEndDate(Date endDate) {
+        this.endDate = endDate;
+    }
+
+    public void setStartDate(Date startDate) {
+        this.startDate = startDate;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -97,17 +112,12 @@ public long getEntityOwnerId() {
 
     @Override
     public void execute() {
-        if (ids == null && type == null && endDate == null) {
-            throw new InvalidParameterValueException("either ids, type or enddate must be specified");
-        } else if (startDate != null && endDate == null) {
-            throw new InvalidParameterValueException("enddate must be specified with startdate parameter");
-        }
         boolean result = _mgr.archiveEvents(this);
         if (result) {
             SuccessResponse response = new SuccessResponse(getCommandName());
             setResponseObject(response);
-        } else {
-            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to archive Events, one or more parameters has invalid values");
+            return;
         }
+        throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to archive Events. One or more parameters have invalid values.");
     }
 }

api/src/main/java/org/apache/cloudstack/api/command/user/event/DeleteEventsCmd.java

@@ -31,7 +31,6 @@
 import org.apache.cloudstack.context.CallContext;
 
 import com.cloud.event.Event;
-import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.user.Account;
 
 @APICommand(name = "deleteEvents", description = "Delete one or more events.", responseObject = SuccessResponse.class, entityType = {Event.class},
@@ -82,6 +81,22 @@ public String getType() {
         return type;
     }
 
+    public void setIds(List<Long> ids) {
+        this.ids = ids;
+    }
+
+    public void setEndDate(Date endDate) {
+        this.endDate = endDate;
+    }
+
+    public void setStartDate(Date startDate) {
+        this.startDate = startDate;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
     // ///////////////////////////////////////////////////
     // ///////////// API Implementation///////////////////
     // ///////////////////////////////////////////////////
@@ -97,17 +112,12 @@ public long getEntityOwnerId() {
 
     @Override
     public void execute() {
-        if (ids == null && type == null && endDate == null) {
-            throw new InvalidParameterValueException("either ids, type or enddate must be specified");
-        } else if (startDate != null && endDate == null) {
-            throw new InvalidParameterValueException("enddate must be specified with startdate parameter");
-        }
         boolean result = _mgr.deleteEvents(this);
         if (result) {
             SuccessResponse response = new SuccessResponse(getCommandName());
             setResponseObject(response);
         } else {
-            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to delete Events, one or more parameters has invalid values");
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to delete any events. One or more parameters have invalid values.");
         }
     }
 }

api/src/main/java/org/apache/cloudstack/api/command/user/event/ListEventsCmd.java

@@ -134,7 +134,6 @@ public String getState() {
 
     @Override
     public void execute() {
-
         ListResponse<EventResponse> response = _queryService.searchForEvents(this);
         response.setResponseName(getCommandName());
         setResponseObject(response);

engine/schema/src/main/java/com/cloud/event/dao/EventDao.java

@@ -20,19 +20,12 @@
 import java.util.List;
 
 import com.cloud.event.EventVO;
-import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericDao;
-import com.cloud.utils.db.SearchCriteria;
 
 public interface EventDao extends GenericDao<EventVO, Long> {
-    public List<EventVO> searchAllEvents(SearchCriteria<EventVO> sc, Filter filter);
-
-    public List<EventVO> listOlderEvents(Date oldTime);
-
-    EventVO findCompletedEvent(long startId);
-
-    public List<EventVO> listToArchiveOrDeleteEvents(List<Long> ids, String type, Date startDate, Date endDate, List<Long> accountIds);
-
-    public void archiveEvents(List<EventVO> events);
+    long archiveEvents(List<Long> ids, String type, Date startDate, Date endDate, Long accountId, List<Long> domainIds,
+                       long limitPerQuery);
 
+    long purgeAll(List<Long> ids, Date startDate, Date endDate, Date limitDate, String type, Long accountId, List<Long> domainIds,
+                  long limitPerQuery);
 }

engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java

@@ -16,15 +16,20 @@
 // under the License.
 package com.cloud.event.dao;
 
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
 import java.util.Date;
 import java.util.List;
+import java.util.stream.Collectors;
 
 
+import com.cloud.utils.db.DB;
+import com.cloud.utils.db.Filter;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Component;
 
-import com.cloud.event.Event.State;
 import com.cloud.event.EventVO;
-import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
@@ -33,83 +38,90 @@
 
 @Component
 public class EventDaoImpl extends GenericDaoBase<EventVO, Long> implements EventDao {
-    protected final SearchBuilder<EventVO> CompletedEventSearch;
+
     protected final SearchBuilder<EventVO> ToArchiveOrDeleteEventSearch;
 
     public EventDaoImpl() {
-        CompletedEventSearch = createSearchBuilder();
-        CompletedEventSearch.and("state", CompletedEventSearch.entity().getState(), SearchCriteria.Op.EQ);
-        CompletedEventSearch.and("startId", CompletedEventSearch.entity().getStartId(), SearchCriteria.Op.EQ);
-        CompletedEventSearch.and("archived", CompletedEventSearch.entity().getArchived(), Op.EQ);
-        CompletedEventSearch.done();
-
         ToArchiveOrDeleteEventSearch = createSearchBuilder();
+        ToArchiveOrDeleteEventSearch.select("id", SearchCriteria.Func.NATIVE, ToArchiveOrDeleteEventSearch.entity().getId());
         ToArchiveOrDeleteEventSearch.and("id", ToArchiveOrDeleteEventSearch.entity().getId(), Op.IN);
         ToArchiveOrDeleteEventSearch.and("type", ToArchiveOrDeleteEventSearch.entity().getType(), Op.EQ);
-        ToArchiveOrDeleteEventSearch.and("accountIds", ToArchiveOrDeleteEventSearch.entity().getAccountId(), Op.IN);
+        ToArchiveOrDeleteEventSearch.and("accountId", ToArchiveOrDeleteEventSearch.entity().getAccountId(), Op.EQ);
+        ToArchiveOrDeleteEventSearch.and("domainIds", ToArchiveOrDeleteEventSearch.entity().getDomainId(), Op.IN);
         ToArchiveOrDeleteEventSearch.and("createdDateB", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.BETWEEN);
         ToArchiveOrDeleteEventSearch.and("createdDateL", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.LTEQ);
+        ToArchiveOrDeleteEventSearch.and("createdDateLT", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.LT);
         ToArchiveOrDeleteEventSearch.and("archived", ToArchiveOrDeleteEventSearch.entity().getArchived(), Op.EQ);
         ToArchiveOrDeleteEventSearch.done();
     }
 
-    @Override
-    public List<EventVO> searchAllEvents(SearchCriteria<EventVO> sc, Filter filter) {
-        return listIncludingRemovedBy(sc, filter);
-    }
-
-    @Override
-    public List<EventVO> listOlderEvents(Date oldTime) {
-        if (oldTime == null)
-            return null;
-        SearchCriteria<EventVO> sc = createSearchCriteria();
-        sc.addAnd("createDate", SearchCriteria.Op.LT, oldTime);
-        sc.addAnd("archived", SearchCriteria.Op.EQ, false);
-        return listIncludingRemovedBy(sc, null);
-    }
-
-    @Override
-    public EventVO findCompletedEvent(long startId) {
-        SearchCriteria<EventVO> sc = CompletedEventSearch.create();
-        sc.setParameters("state", State.Completed);
-        sc.setParameters("startId", startId);
-        sc.setParameters("archived", false);
-        return findOneIncludingRemovedBy(sc);
-    }
-
-    @Override
-    public List<EventVO> listToArchiveOrDeleteEvents(List<Long> ids, String type, Date startDate, Date endDate, List<Long> accountIds) {
+    private SearchCriteria<EventVO> createEventSearchCriteria(List<Long> ids, String type, Date startDate, Date endDate,
+                                                              Date limitDate, Long accountId, List<Long> domainIds) {
         SearchCriteria<EventVO> sc = ToArchiveOrDeleteEventSearch.create();
-        if (ids != null) {
-            sc.setParameters("id", ids.toArray(new Object[ids.size()]));
+
+        if (CollectionUtils.isNotEmpty(ids)) {
+            sc.setParameters("id", ids.toArray(new Object[0]));
         }
-        if (type != null) {
-            sc.setParameters("type", type);
+        if (CollectionUtils.isNotEmpty(domainIds)) {
+            sc.setParameters("domainIds", domainIds.toArray(new Object[0]));
         }
         if (startDate != null && endDate != null) {
             sc.setParameters("createdDateB", startDate, endDate);
         } else if (endDate != null) {
             sc.setParameters("createdDateL", endDate);
         }
-        if (accountIds != null && !accountIds.isEmpty()) {
-            sc.setParameters("accountIds", accountIds.toArray(new Object[accountIds.size()]));
-        }
+        sc.setParametersIfNotNull("accountId", accountId);
+        sc.setParametersIfNotNull("createdDateLT", limitDate);
+        sc.setParametersIfNotNull("type", type);
         sc.setParameters("archived", false);
-        return search(sc, null);
+
+        return sc;
     }
 
     @Override
-    public void archiveEvents(List<EventVO> events) {
-        if (events != null && !events.isEmpty()) {
-            TransactionLegacy txn = TransactionLegacy.currentTxn();
-            txn.start();
-            for (EventVO event : events) {
-                event = lockRow(event.getId(), true);
-                event.setArchived(true);
-                update(event.getId(), event);
-                txn.commit();
+    public long archiveEvents(List<Long> ids, String type, Date startDate, Date endDate, Long accountId, List<Long> domainIds,
+                              long limitPerQuery) {
+        SearchCriteria<EventVO> sc = createEventSearchCriteria(ids, type, startDate, endDate, null, accountId, domainIds);
+        Filter filter = null;
+        if (limitPerQuery > 0) {
+            filter = new Filter(limitPerQuery);
+        }
+
+        long archived;
+        long totalArchived = 0L;
+
+        do {
+            List<EventVO> events = search(sc, filter);
+            if (events.isEmpty()) {
+                break;
             }
-            txn.close();
+
+            archived = archiveEventsInternal(events);
+            totalArchived += archived;
+        } while (limitPerQuery > 0 && archived >= limitPerQuery);
+
+        return totalArchived;
+    }
+
+    @DB
+    private long archiveEventsInternal(List<EventVO> events) {
+        final String idsAsString = events.stream()
+                .map(e -> Long.toString(e.getId()))
+                .collect(Collectors.joining(","));
+        final String query = String.format("UPDATE event SET archived=true WHERE id IN (%s)", idsAsString);
+
+        try (TransactionLegacy txn = TransactionLegacy.currentTxn();
+             PreparedStatement pstmt = txn.prepareStatement(query)) {
+            return pstmt.executeUpdate();
+        } catch (SQLException e) {
+            throw new CloudRuntimeException(e);
         }
     }
+
+    @Override
+    public long purgeAll(List<Long> ids, Date startDate, Date endDate, Date limitDate, String type, Long accountId,
+                         List<Long> domainIds, long limitPerQuery) {
+        SearchCriteria<EventVO> sc = createEventSearchCriteria(ids, type, startDate, endDate, limitDate, accountId, domainIds);
+        return batchExpunge(sc, limitPerQuery);
+    }
 }

engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java

@@ -25,7 +25,10 @@ public class DbUpgradeUtils {
 
     public static void addIndexIfNeeded(Connection conn, String tableName, String... columnNames) {
         String indexName = dao.generateIndexName(tableName, columnNames);
+        addIndexWithNameIfNeeded(conn, tableName, indexName, columnNames);
+    }
 
+    public static void addIndexWithNameIfNeeded(Connection conn, String tableName, String indexName, String... columnNames) {
         if (!dao.indexExists(conn, tableName, indexName)) {
             dao.createIndex(conn, tableName, indexName, columnNames);
         }

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java

@@ -17,6 +17,7 @@
 package com.cloud.upgrade.dao;
 
 import java.io.InputStream;
+import java.sql.Connection;
 
 import com.cloud.utils.exception.CloudRuntimeException;
 
@@ -42,4 +43,15 @@ public InputStream[] getPrepareScripts() {
 
         return new InputStream[] {script};
     }
+
+    @Override
+    public void performDataMigration(Connection conn) {
+        addIndexes(conn);
+    }
+
+    private void addIndexes(Connection conn) {
+        DbUpgradeUtils.addIndexWithNameIfNeeded(conn, "event", "i_event__multiple_columns_for_generic_search",
+                "account_id", "domain_id", "archived",  "display", "resource_type", "resource_id", "start_id", "type", "level",
+                "created", "id");
+    }
 }

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -920,7 +920,7 @@ private Pair<List<Long>, Integer> searchForEventIdsAndCount(ListEventsCmd cmd) {
         searchFilter.addOrderBy(EventVO.class, "id", false);
 
         SearchBuilder<EventVO> eventSearchBuilder = eventDao.createSearchBuilder();
-        eventSearchBuilder.select(null, Func.DISTINCT, eventSearchBuilder.entity().getId());
+        eventSearchBuilder.select(null, Func.NATIVE, eventSearchBuilder.entity().getId());
         accountMgr.buildACLSearchBuilder(eventSearchBuilder, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
 
         eventSearchBuilder.and("id", eventSearchBuilder.entity().getId(), SearchCriteria.Op.EQ);

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -537,7 +537,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
     public static final ConfigKey<Long> DELETE_QUERY_BATCH_SIZE = new ConfigKey<>("Advanced", Long.class, "delete.query.batch.size", "0",
             "Indicates the limit applied while deleting entries in bulk. With this, the delete query will apply the limit as many times as necessary," +
                     " to delete all the entries. This is advised when retaining several days of records, which can lead to slowness. <= 0 means that no limit will " +
-                    "be applied. Default value is 0. For now, this is used for deletion of vm & volume stats only.", true);
+                    "be applied. Default value is 0. For now, this is used for deletion of VM stats, volume stats and events.", true);
 
     private static final String IOPS_READ_RATE = "IOPS Read";
     private static final String IOPS_WRITE_RATE = "IOPS Write";