Memory Profiler w/ Descriptions in README.md and Sample Output

Hoffman2/pris

@@ -124,8 +124,19 @@ if len(mxnames) is not 0:
         linbreg.A = joblib.load(s)
     #
     pypris = loadPyPRIS(path, 'PyPRIS_pris{}'.format(linbreg.PyPRIS_iter[4:]))
+
     # set a check mark for pypris:
     pypris.set_check_mark()
+
+    # ----------- **** Memory Profiler Setting for Existing PRIS *** ------------
+
+    pypris.memory_profiler = True
+
+    # ---------------------------------------------------------------------------
+
+    if pypris.memory_profiler is True:
+        tracemalloc.start()
+
     # put the configured observer to the pypris to perform the tasks of 'observe_biplane'
     # when a pypris object observes, it is through the observer and it is
     # done with the observer's observe_biplane skill (skill = method).
@@ -145,6 +156,9 @@ if len(mxnames) is not 0:
     print("---------------- PRIS refinement #" + str(Iter) + " ------------------")
     print("")
     linbreg.go()
+    if pypris.memory_profiler is True:
+        snapshot = tracemalloc.take_snapshot()
+        pypris.display_top(snapshot)
 else:
     print("PyPRIS Begin!")
     print("")
@@ -158,6 +172,16 @@ else:
     #
     # get a PyPRIS object:
     pypris = PyPRIS()
+
+    # ----------- **** Memory Profiler Setting for New PRIS *** ------------
+
+    pypris.memory_profiler = True
+
+    # ----------------------------------------------------------------------
+
+    if pypris.memory_profiler is True:
+        tracemalloc.start()
+
     pypris.observation = blur.ravel()
     pypris.current_candidates_intervals = ticket.init_candidates_intervals  # initialize the first intervals of neighboring candidate voxels.
     #
@@ -217,6 +241,11 @@ else:
     linbreg.get_ready()
     linbreg.go()
     Iter=0
+
+    # Saves top 10 memory snapshots at current PyPRIS iteration into saved_objects folder 
+    if pypris.memory_profiler is True:
+        snapshot = tracemalloc.take_snapshot()
+        pypris.display_top(snapshot)
 
 # now continue the pris iterations
 # construct sensing matrix
@@ -263,3 +292,7 @@ for PRIS_iter in np.arange(Iter + 1, ticket.PRIS_iter_end):
     # recover
     linbreg.get_ready()
     linbreg.go()
+
+    if pypris.memory_profiler is True:
+        snapshot = tracemalloc.take_snapshot()
+        pypris.display_top(snapshot)

PyPRIS/PRIS.py

@@ -4,6 +4,10 @@
 import pickle
 import joblib
 import matplotlib
+from collections import Counter
+import linecache
+import os
+import tracemalloc
 
 try:
     from matplotlib import pyplot as plt
@@ -36,6 +40,33 @@ def __init__(self):
         self.ifsave = True
         self.path_s = "./saved_objects"
         self.expansion = False
+        self.memory_profiler = False
+
+    def display_top(self, snapshot, key_type='lineno', limit=10):
+        with open("{}/PyPRIS_pris{}_mem.file \n".format(self.path_s, self.current_PRIS_ItN), "w") as f:
+            snapshot = snapshot.filter_traces((
+                tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
+                tracemalloc.Filter(False, "<unknown>"),
+            ))
+            top_stats = snapshot.statistics(key_type)
+
+            f.write("Top %s lines \n" % limit)
+            for index, stat in enumerate(top_stats[:limit], 1):
+                frame = stat.traceback[0]
+                # replace "/path/to/module/file.py" with "module/file.py"
+                filename = os.sep.join(frame.filename.split(os.sep)[-2:])
+                f.write("#%s: %s:%s: %.1f KiB \n"
+              % (index, filename, frame.lineno, stat.size / 1024))
+                line = linecache.getline(frame.filename, frame.lineno).strip()
+                if line:
+                    f.write('    %s \n' % line)
+
+            other = top_stats[limit:]
+            if other:
+                size = sum(stat.size for stat in other)
+                f.write("%s other: %.1f KiB \n" % (len(other), size / 1024))
+            total = sum(stat.size for stat in top_stats)
+            f.write("Total allocated size: %.1f KiB \n" % (total / 1024))
 
     def save(self):
         import os

README.md

@@ -24,7 +24,7 @@ Before running notebook files, set up the environment of Ipython kernel as well
 Finally, after starting jupyter notebook, switch kernel to current environment by clicking "Kernel -> Change kernel -> PyPRIS_env".
 
 ## On Hoffman2:
-Upload: upload test_data, PyPRIS, PyPRIS_env.yml and \*.py files to hoffman2 under your home directory.
+Upload: upload the entire test_data and PyPRIS folders, *PyPRIS_env.yml* and *\*.py* files to hoffman2 under your home directory.
 
 #### configure environment for PyPRIS (PyPRIS_env) on Hoffman2 from terminal (recommended terminals include MobaXTerm and Putty) 
 After log-in from the terminal, start an interactive session with: `qrsh` (this step may take a few minutes)
@@ -33,7 +33,7 @@ Load anaconda with:
 
 `module load anaconda`
 
-Install the PyPRIS_env enviroenemt to your home directory with:
+Install the PyPRIS_env environment to your home directory with:
 
 `conda env create -f PyPRIS_env.yml` (This step may take about 10 minutes)
 
@@ -46,21 +46,25 @@ To log-out from the PyPRIS environment, type:
 `conda deactivate`
 
 ### Computation on Hoffman2
-* upload the observation files (for example: blur_plane1.tif, blur_plane7.tif and psf.tif in test_data) to hoffman2. keep both files under the same file folder 
-* prepare tickets by updating and executing prep_for_hoffman2.py locally (recommended.)
-Examples include prep_for_hoffman2_Feature2.py and prep_for_hoffman2_feature3.py
-  * specify file names and file paths for each plane.
-  * specify ticket.ticket_folder
-  * specify index range for x,y,z dimensions (the range need to match your fov size)
-  * set a wide range for bgCSF, mu, and alpha values, in the form of absolute values.
-  * upload to hoffman2.
-  * put in the pris and pris.Ini files into each ticket folder.
-  * To ensure the pris file, the ticket files and the associated file folders are readable, writable and executable, use `chmod -R u+rwx * `
-  * **Perform Single Job test!** extremley important. test with a single job first. Because if you fail a big batch of jobs, your priority will be reduced in the queue.
-      * if any error arises, open an interactive session to debug on the server by activating an interactive python session. To initiate an interactive python session from the terminal on hoffman2 (recommended to use Putty), type `Python`
-      * fix bugs until a linbreg recovery can be successfully executed in the interactive session.
-      * **perform single JOb test before moving on!**
-      * test with `python pris` from the terminal.
-  * submit batch jobs
-  * remember that there are complications associated with files from Github that was pushed from a windows system!
+* Upload the observation files (e.g. *blur_plane1.tif*, *blur_plane7.tif and *psf.tif* in test_data) to Hoffman2. Keep both files under the same root folder. 
+* Prepare tickets by updating and executing *prep_for_hoffman2.py* locally (recommended.)
+Examples include *prep_for_hoffman2_Feature2.py* and *prep_for_hoffman2_feature3.py*
+  * Specify file names and file paths for each plane.
+  * Specify ticket.ticket_folder
+  * Specify index range for x,y,z dimensions (the range need to match your fov size)
+  * Set a wide range for bgCSF, mu, and alpha values, in the form of absolute values.
+  * Upload the entire ticket folders to Hoffman2.
+  * Copy pris and pris.Ini files into each ticket folder.
+  * Turn on or off **memory profiling options** in *pris* file 
+      * Line 133: if a previously saved PyPRIS object already exists 
+      * Line 178: for new calculations
+      * Top 10 memory usages by line for each PyPRIS iteration will be saved in the saved_objects folder under the name *PyPRIS_\*_mem.file*. To see a sample output, please check *sample_memprof_output.file*.
+  * To ensure the *pris* file, the ticket files and the associated file folders are readable, writable and executable, use `chmod -R u+rwx * `
+  * **Perform single job test!** This is extremley important. Test with a single job first, as if you fail a big batch of jobs, your priority will be reduced in the queue.
+      * If any error arises, open an interactive session to debug on the server by activating an interactive Python session. To initiate such session from the terminal on Hoffman2 (recommended to use PuTTY), type `Python`
+      * Fix bugs until a linbreg recovery can be successfully executed in the interactive session.
+      * **Perform single job test before moving on!**
+      * Test with `python pris` from the terminal.
+  * Submit batch jobs.
+  * Remember that there are complications associated with files from Github that was pushed from a Windows system!
       * Temporary solution: creat the same file on the linux system by copy-pasting the content in to a new file with the same file name (instead of copy-pasting the file).

sample_memprof_output.file

@@ -0,0 +1,23 @@
+Top 10 lines 
+#1: joblib/numpy_pickle.py:141: 10236774.6 KiB 
+    array = unpickler.np.empty(count, dtype=self.dtype) 
+#2: PyPRIS/PRIS.py:616: 2217.1 KiB 
+    self.bg.append(self.x[self.x.size - 1]) 
+#3: PyPRIS/PRIS.py:613: 2217.1 KiB 
+    /self.hist_res[it_count - 1] \ 
+#4: PyPRIS/PRIS.py:610: 2217.1 KiB 
+    self.hist_delta_res.append((self.hist_res[it_count] - self.hist_res[it_count - 1])) 
+#5: PyPRIS/PRIS.py:341: 1585.9 KiB 
+    self.it_count += 1 
+#6: linalg/linalg.py:2360: 1495.3 KiB 
+    ret = sqrt(sqnorm) 
+#7: PyPRIS/PRIS.py:377: 1015.9 KiB 
+    self.x = self.alpha * self.shrink(self.x) 
+#8: PyPRIS/PRIS.py:350: 1015.4 KiB 
+    self.respj = np.dot(self.res, self.A) 
+#9: core/numeric.py:203: 1015.4 KiB 
+    a = empty(shape, dtype, order) 
+#10: PyPRIS/PRIS.py:609: 858.2 KiB 
+    self.hist_res.append(np.linalg.norm(res)) 
+44 other: 1827.4 KiB 
+Total allocated size: 10252239.4 KiB