Issue 695: Add feature to allow unreferenced files to be deleted from apps

api/src/org/labkey/api/exp/api/ExperimentService.java

@@ -60,6 +60,7 @@
 import org.labkey.api.exp.query.ExpRunTable;
 import org.labkey.api.exp.query.ExpSampleTypeTable;
 import org.labkey.api.exp.query.ExpSchema;
+import org.labkey.api.exp.query.ExpUnreferencedSampleFilesTable;
 import org.labkey.api.exp.query.SampleStatusTable;
 import org.labkey.api.gwt.client.AuditBehaviorType;
 import org.labkey.api.gwt.client.model.GWTDomain;
@@ -81,7 +82,6 @@
 import org.labkey.api.util.Pair;
 import org.labkey.api.util.StringUtilsLabKey;
 import org.labkey.api.view.HttpView;
-import org.labkey.api.view.NotFoundException;
 import org.labkey.api.view.ViewBackgroundInfo;
 import org.labkey.api.view.ViewContext;
 import org.labkey.vfs.FileLike;
@@ -670,6 +670,8 @@ static void validateParentAlias(Map<String, String> aliasMap, Set<String> reserv
 
     SampleStatusTable createSampleStatusTable(ExpSchema expSchema, ContainerFilter cf);
 
+    ExpUnreferencedSampleFilesTable createUnreferencedSampleFilesTable(ExpSchema expSchema, ContainerFilter cf);
+
     FilteredTable<ExpSchema> createFieldsTable(ExpSchema expSchema, ContainerFilter cf);
 
     FilteredTable<ExpSchema> createPhiFieldsTable(ExpSchema expSchema, ContainerFilter cf);

api/src/org/labkey/api/exp/query/ExpDataTable.java

@@ -35,6 +35,7 @@ enum Column
         SourceProtocolApplication,
         SourceApplicationInput,
         DataFileUrl,
+        ReferenceCount,
         Run,
         RunApplication,
         RunApplicationOutput,

api/src/org/labkey/api/exp/query/ExpSchema.java

@@ -47,6 +47,7 @@
 import org.labkey.api.security.User;
 import org.labkey.api.security.permissions.InsertPermission;
 import org.labkey.api.security.permissions.ReadPermission;
+import org.labkey.api.settings.AppProps;
 import org.labkey.api.util.StringExpression;
 import org.labkey.api.view.ActionURL;
 import org.labkey.api.view.ViewContext;
@@ -70,6 +71,7 @@ public class ExpSchema extends AbstractExpSchema
     public static final String SAMPLE_STATE_TYPE_TABLE = "SampleStateType";
     public static final String SAMPLE_TYPE_CATEGORY_TABLE = "SampleTypeCategoryType";
     public static final String MEASUREMENT_UNITS_TABLE = "MeasurementUnits";
+    public static final String SAMPLE_FILES_TABLE = "UnreferencedSampleFiles";
 
     public static final SchemaKey SCHEMA_EXP = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME);
     public static final SchemaKey SCHEMA_EXP_DATA = SchemaKey.fromString(SCHEMA_EXP, ExpSchema.NestedSchemas.data.name());
@@ -220,6 +222,20 @@ public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFil
                 return expSchema.setupTable(result);
             }
         },
+        UnreferencedSampleFiles
+        {
+             @Override
+             public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf)
+             {
+                 return ExperimentService.get().createUnreferencedSampleFilesTable(expSchema, cf);
+             }
+
+            @Override
+            public boolean includeTable()
+            {
+                return AppProps.getInstance().isOptionalFeatureEnabled(SAMPLE_FILES_TABLE);
+            }
+        },
         SampleStatus
         {
             @Override

api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java

@@ -0,0 +1,8 @@
+package org.labkey.api.exp.query;
+
+import org.labkey.api.data.ContainerFilterable;
+import org.labkey.api.data.TableInfo;
+
+public interface ExpUnreferencedSampleFilesTable extends ContainerFilterable, TableInfo
+{
+}

api/src/org/labkey/api/files/FileContentService.java

@@ -325,6 +325,8 @@ default void fireFileDeletedEvent(@NotNull Path deleted, @Nullable User user, @N
      */
     SQLFragment listFilesQuery(@NotNull User currentUser);
 
+    SQLFragment listSampleFilesQuery(@NotNull User currentUser);
+
     void setWebfilesEnabled(boolean enabled, User user);
 
     /**
@@ -345,6 +347,14 @@ enum PathType { full, serverRelative, folderRelative }
      */
     void ensureFileData(@NotNull ExpDataTable table);
 
+    /**
+     * Fix the container column in the exp.data table for files that were moved as part of a sample move operation
+     * but did not have their containers updated
+     * @param admin The user doing the repair
+     * @return Number of duplicate rows removed from exp.data table
+     */
+    int fixContainerForExpDataFiles(User admin);
+
     /**
      * Allows a module to register a directory pattern to be checked in the files webpart in order to zip the matching directory before uploading.
      * @param directoryPattern DirectoryPattern

api/src/org/labkey/api/files/FileListener.java

@@ -94,4 +94,9 @@ default void fileDeleted(@NotNull Path deleted, @Nullable User user, @Nullable C
      * </ul>
      */
     SQLFragment listFilesQuery();
+
+    @Nullable default SQLFragment listSampleFilesQuery()
+    {
+        return null;
+    }
 }

api/src/org/labkey/api/files/TableUpdaterFileListener.java

@@ -353,10 +353,10 @@ public Collection<File> listFiles(@Nullable Container container)
     @Override
     public SQLFragment listFilesQuery()
     {
-        return listFilesQuery(false, null);
+        return listFilesQuery(false, null, false);
     }
 
-    public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath)
+    public SQLFragment listFilesQuery(boolean skipCreatedModified, CharSequence filePath, boolean extractName)
     {
         SQLFragment selectFrag = new SQLFragment();
         selectFrag.append("SELECT\n");
@@ -395,6 +395,16 @@ else if (_table.getColumn("Folder") != null)
 
         selectFrag.append("  ").appendIdentifier(_pathColumn.getSelectIdentifier()).append(" AS FilePath,\n");
 
+        if (extractName)
+        {
+            SqlDialect dialect = _table.getSchema().getSqlDialect();
+            SQLFragment fileNameFrag = new SQLFragment();
+            fileNameFrag.append("regexp_replace(").appendIdentifier(_pathColumn.getSelectIdentifier()).append(", ");
+            fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral(".*/")).append(", ");
+            fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral("")).append(")");;
+            selectFrag.append("  ").append(fileNameFrag).append(" AS FilePathShort,\n");
+        }
+
         if (_keyColumn != null)
             selectFrag.append("  ").appendIdentifier(_keyColumn.getSelectIdentifier()).append(" AS SourceKey,\n");
         else
@@ -408,6 +418,8 @@ else if (_table.getColumn("Folder") != null)
 
         if (StringUtils.isEmpty(filePath))
             selectFrag.append(" IS NOT NULL\n");
+        else if (filePath instanceof SQLFragment)
+            selectFrag.append(" = ").append(filePath).append("\n");
         else
             selectFrag.append(" = ").appendStringLiteral(filePath, _table.getSchema().getSqlDialect()).append("\n");
 

experiment/src/org/labkey/experiment/ExperimentModule.java

@@ -18,6 +18,7 @@
 import org.apache.commons.lang3.math.NumberUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.json.JSONObject;
 import org.labkey.api.admin.FolderSerializationRegistry;
 import org.labkey.api.assay.AssayProvider;
 import org.labkey.api.assay.AssayService;
@@ -112,6 +113,7 @@
 import org.labkey.api.vocabulary.security.DesignVocabularyPermission;
 import org.labkey.api.webdav.WebdavResource;
 import org.labkey.api.webdav.WebdavService;
+import org.labkey.api.writer.ContainerUser;
 import org.labkey.experiment.api.DataClassDomainKind;
 import org.labkey.experiment.api.ExpDataClassImpl;
 import org.labkey.experiment.api.ExpDataClassTableImpl;
@@ -181,6 +183,7 @@
 import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI;
 import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI;
 import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME;
+import static org.labkey.api.exp.query.ExpSchema.SAMPLE_FILES_TABLE;
 
 public class ExperimentModule extends SpringModule
 {
@@ -266,6 +269,9 @@ protected void init()
         }
         else
         {
+            OptionalFeatureService.get().addExperimentalFeatureFlag(SAMPLE_FILES_TABLE, "Manage Unreferenced Sample Files",
+                    "Enable 'Unreferenced Sample Files' table to view and delete sample files that are no longer referenced by samples", false);
+
             OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression",
                 "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true);
         }
@@ -1117,4 +1123,12 @@ public Collection<String> getProvisionedSchemaNames()
     {
         return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME);
     }
+
+    @Override
+    public JSONObject getPageContextJson(ContainerUser context)
+    {
+        JSONObject json = super.getPageContextJson(context);
+        json.put(SAMPLE_FILES_TABLE, OptionalFeatureService.get().isFeatureEnabled(SAMPLE_FILES_TABLE));
+        return json;
+    }
 }

experiment/src/org/labkey/experiment/FileLinkFileListener.java

@@ -275,7 +275,7 @@ public SQLFragment listFilesQuery(boolean skipCreatedModified)
         return listFilesQuery(skipCreatedModified, null);
     }
 
-    public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath)
+    public SQLFragment listFilesQuery(boolean skipCreatedModified, CharSequence filePath)
     {
         final SQLFragment frag = new SQLFragment();
 
@@ -298,8 +298,11 @@ public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath)
         frag.append("WHERE\n");
         if (StringUtils.isEmpty(filePath))
             frag.append("  op.StringValue IS NOT NULL AND\n");
+        else if (filePath instanceof SQLFragment)
+            frag.append("  op.StringValue = ").append(filePath).append(" AND\n");
         else
             frag.append("  op.StringValue = ").appendStringLiteral(filePath, OntologyManager.getTinfoObject().getSqlDialect()).append(" AND\n");
+
         frag.append("  o.ObjectId = op.ObjectId AND\n");
         frag.append("  PropertyId IN (\n");
         frag.append("    SELECT PropertyId\n");
@@ -311,7 +314,26 @@ public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath)
             SQLFragment containerFrag = new SQLFragment("?", containerId);
             TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, null, containerFrag);
             frag.append("UNION").append(StringUtils.isEmpty(filePath) ? "" : " ALL" /*keep duplicate*/).append("\n");
-            frag.append(updater.listFilesQuery(skipCreatedModified, filePath));
+            frag.append(updater.listFilesQuery(skipCreatedModified, filePath, false));
+        });
+
+        return frag;
+    }
+
+    @Override
+    public SQLFragment listSampleFilesQuery()
+    {
+        final SQLFragment frag = new SQLFragment();
+
+        hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> {
+            if (PROVISIONED_SCHEMA_NAME.equalsIgnoreCase(schema.getName()))
+            {
+                SQLFragment containerFrag = new SQLFragment("?", containerId);
+                TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, "rowid", containerFrag);
+                if (!frag.isEmpty())
+                    frag.append("UNION").append("").append("\n");
+                frag.append(updater.listFilesQuery(true, null, true));
+            }
         });
 
         return frag;

experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java

@@ -34,12 +34,14 @@
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerFilter;
 import org.labkey.api.data.ContainerManager;
+import org.labkey.api.data.CoreSchema;
 import org.labkey.api.data.ExcelWriter;
 import org.labkey.api.data.JdbcType;
 import org.labkey.api.data.MutableColumnInfo;
 import org.labkey.api.data.RenderContext;
 import org.labkey.api.data.SQLFragment;
 import org.labkey.api.data.SimpleFilter;
+import org.labkey.api.data.SqlSelector;
 import org.labkey.api.exp.DomainDescriptor;
 import org.labkey.api.exp.OntologyManager;
 import org.labkey.api.exp.PropertyColumn;
@@ -75,6 +77,7 @@
 import org.labkey.api.security.User;
 import org.labkey.api.security.permissions.InsertPermission;
 import org.labkey.api.security.permissions.UpdatePermission;
+import org.labkey.api.settings.AppProps;
 import org.labkey.api.util.ConfigurationException;
 import org.labkey.api.util.FileUtil;
 import org.labkey.api.util.HtmlString;
@@ -84,6 +87,7 @@
 import org.labkey.api.writer.HtmlWriter;
 import org.labkey.api.writer.MemoryVirtualFile;
 import org.labkey.api.writer.VirtualFile;
+import org.labkey.experiment.FileLinkFileListener;
 import org.labkey.experiment.controllers.exp.ExperimentController;
 import org.labkey.experiment.lineage.LineageMethod;
 import org.springframework.beans.MutablePropertyValues;
@@ -200,6 +204,8 @@ public List<String> addFileColumns(boolean isFilesTable)
         addColumn(Column.FileExtension);
         addColumn(Column.WebDavUrl);
         addColumn(Column.WebDavUrlRelative);
+        if (AppProps.getInstance().isOptionalFeatureEnabled(ExpSchema.SAMPLE_FILES_TABLE))
+            addColumn(getFileLinkReferenceCountColumn());
         var flagCol = addColumn(Column.Flag);
         if (isFilesTable)
             flagCol.setLabel("Description");
@@ -227,6 +233,18 @@ public List<String> addFileColumns(boolean isFilesTable)
         return customProps;
     }
 
+    // This is included in exp.data, not just exp.files because we want to be able to show a filtered view of
+    // sample files from our applications, and exp.files will not show subfolders
+    private MutableColumnInfo getFileLinkReferenceCountColumn()
+    {
+        var result = wrapColumn(Column.ReferenceCount.name(), _rootTable.getColumn("RowId"));
+        result.setDescription("The number of references to this file from File fields in any domain.");
+        result.setJdbcType(JdbcType.INTEGER);
+        result.setHidden(true);
+        result.setDisplayColumnFactory(new ReferenceCountDisplayColumnFactory());
+        return result;
+    }
+
     @Override
     public boolean supportTableRules() // intentional override
     {