Fan control (#387)

* Adds smarter fan setup * Fan-control docs and misc doc fixes. * Adds help statement to jackhammer setup-fans * typo * Fix typos in comments / logs --------- Co-authored-by: simonscryo <[email protected]>
simonsobs · Oct 9, 2023 · c7bc197 · c7bc197
1 parent e0aa529
commit c7bc197
Show file tree

Hide file tree

Showing 5 changed files with 114 additions and 22 deletions.
diff --git a/docs/conf.py b/docs/conf.py
@@ -26,6 +26,7 @@
     'sotodlib.tod_ops',
     'sotodlib.core',
     'tqdm.auto'
+    'tqdm',
 ]
 
 from unittest import mock

diff --git a/docs/configs.rst b/docs/configs.rst
@@ -49,8 +49,8 @@ The following parameters can be set in the top-level of the yaml file
      - ID of the smurf-crate, e.g. ``1``
    * - ``shelf_manager``
      - Shelf manager name
-   * - ``max_fan_level``
-     - Max Fan Level to be set on reboot
+   * - ``smurf_fans``
+     - Dict used to set fan-policy and speed on hammer
    * - ``comm_type``
      - Communication type to the crate -- either ``eth`` or ``pcie``
    * - ``meta_register_file``
@@ -59,6 +59,64 @@ The following parameters can be set in the top-level of the yaml file
    * - ``slot_order``
      - A list containing all active slots in the order that they should be 
        started.
+
+Fan Configuration
+```````````````````
+The ``smurf_fans`` key is a dict which can be used to set the fan policy and
+speed during hammer, or with the ``jackhammer setup-fans`` command.
+
+Before configuring you need to determine the address of the fans on the smurf. 
+To do this, ssh into the shelf-manager, and run the following:
+
+.. code-block:: bash
+
+  # clia fans
+  > Pigeon Point Shelf Manager Command Line Interpreter
+  > 20: FRU # 4
+  > Current Level: 14
+  > Minimum Speed Level: 1, Maximum Speed Level: 100 Dynamic minimum fan level: 11
+  > 20: FRU # 3
+  > Current Level: 14
+  > Minimum Speed Level: 1, Maximum Speed Level: 100
+
+This tells us that there are two configurable fans, one with address 20 and FRU-id 4, and the other with address 20 and FRU-id 3.
+This also tells us that the maximum speed of the fan is 100.
+
+Our sys-config entry under ``smurf_fans`` should look like
+
+.. code-block:: yaml
+
+  smurf_fans:
+    addresses:
+      - [<address-1>, <fru_id-1>]
+      - [<address-2>, <fru_id-2>]
+    speed: <speed>
+    policy: <policy>
+
+Here, ``speed`` is the speed it will set all fans to, ``policy`` can be
+``enable`` or ``disable``, where ``enable`` will have the fan-controller
+auto-adjust the speed based on the junction temps, and disable will keep the fan
+speed fixed.
+
+It can be dangerous to disable the fan-controller if you are not setting fans to
+their max speed, as the controller will have no ability to increase the speed to
+keep smurf cool if they need to.
+
+However, keeping the fan speeds fixed at their max speed can be beneficial, as
+fan-speed changes cause sudden changes FPGA temp and electronics temp, which make its way into our data.
+
+For operating the SAT (and for most SO smurfs), the max-fan speed is 100, so our configuration looks like:
+
+.. code-block:: yaml
+
+  smurf_fans:
+    addresses:
+      - [20, 3]
+      - [20, 4]
+    speed: 100
+    policy: disable
+
+
 
 Slot Configuration
 ````````````````````

diff --git a/docs/operations/setup.rst b/docs/operations/setup.rst
@@ -341,11 +341,13 @@ Debugging Issues
 
 There are many issues that result in failure somewhere in the setup or relock procedure,
 including:
+
  - Amplifiers not functioning properly
  - Flux-ramp not being received by the UFM
  - SMuRF's resonance center frequency is off of the true resonance frequency
  - SMuRF's eta estimation is off
  - Phase-delay estimation is off
+
 Any of these may result in bad tracking performance and high noise levels.
 
 In order to debug, it is extremely useful to visualize the resonator response

diff --git a/hammers/jackhammer b/hammers/jackhammer
@@ -351,24 +351,7 @@ def hammer_func(args):
 
         # Sets fan levels on crate
         cprint("Setting fan levels", style=TermColors.HEADER)
-
-        min_fan_level = sys_config.get('min_fan_level')
-        init_fan_level = sys_config.get('init_fan_level')
-
-        if (min_fan_level is None) or (init_fan_level is None):
-            # Run with old way, using max_fan_level to set both minfanlevel and
-            # init
-            min_fan_level = sys_config['max_fan_level']
-            init_fan_level = sys_config['max_fan_level']
-            cprint("Using max_fan_level sys_config value to set minfanlevel and "
-                   "initial level. If you've switched to a Comtel crate, please "
-                   "change your sys_config to use the `min_fan_level` and "
-                   "`init_fan_level` variables", style=TermColors.WARNING)
-
-        cmd = f'clia minfanlevel {min_fan_level}; '
-        cmd += f'clia setfanlevel all {init_fan_level}'
-        print(f"Setting crate fans to {init_fan_level}...")
-        run_on_shelf_manager(cmd)
+        setup_fans()
 
     if reboot:
         cprint(f"Rebooting slots: {slots}", style=TermColors.HEADER)
@@ -535,6 +518,46 @@ def activate_func(args):
 def write_env_func(args):
     write_docker_env()
 
+def setup_fans():
+    if 'smurf_fans' not in sys_config:
+        # Old way of setting up fans
+        cprint(
+            "'smurf_fans' key not found in sys config. Setting fan levels using "
+            "'max_fan_level', however this allows fan speed to change over "
+            "time."
+        )
+
+        min_fan_level = sys_config.get('min_fan_level')
+        init_fan_level = sys_config.get('init_fan_level')
+
+        if (min_fan_level is None) or (init_fan_level is None):
+            # Run with old way, using max_fan_level to set both minfanlevel and
+            # init
+            min_fan_level = sys_config['max_fan_level']
+            init_fan_level = sys_config['max_fan_level']
+            cprint("Using max_fan_level sys_config value to set minfanlevel and "
+                   "initial level. If you've switched to a Comtel crate, please "
+                   "change your sys_config to use the `min_fan_level` and "
+                   "`init_fan_level` variables", style=TermColors.WARNING)
+
+        cmd = f'clia minfanlevel {min_fan_level}; '
+        cmd += f'clia setfanlevel all {init_fan_level}'
+        print(f"Setting crate fans to {init_fan_level}...")
+        run_on_shelf_manager(cmd)
+        return
+
+    fans = sys_config['smurf_fans']
+    cmd = ''
+    for addr in fans['addresses']:
+        a, b = addr
+
+        cmd += f"clia setfanpolicy {a} {b} {fans['policy']}; "
+    cmd += f"clia setfanlevel all {fans['speed']};"
+    run_on_shelf_manager(cmd)
+
+# Entrypoint for jackhammer setup-fans
+def setup_fans_func(args):
+    setup_fans()
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
@@ -580,7 +603,7 @@ if __name__ == '__main__':
     setup_parser.set_defaults(func=setup_func)
     setup_parser.add_argument('slots', nargs='*', type=int,
                                help="Specifies the slots to setup")
-    
+
 
     ########### Jackhammer logs parser ############
     log_parser = subparsers.add_parser('logs')
@@ -623,9 +646,17 @@ if __name__ == '__main__':
     write_env_parser = subparsers.add_parser('write-env', help='writes docker-env to .env file')
     write_env_parser.set_defaults(func=write_env_func)
 
+    ########### Jackhammer start-sync parser ###########
     start_sync_parser = subparsers.add_parser('start-sync')
     start_sync_parser.set_defaults(func=start_sync_func)
 
+    ########### Jackhammer setup-fans parser ###########
+    start_sync_parser = subparsers.add_parser(
+        'setup-fans', 
+        help="Sets fan speeds and policy on crate based on sys-config"
+    )
+    start_sync_parser.set_defaults(func=setup_fans_func)
+
     args = parser.parse_args()
     if hasattr(args, 'func'):
         args.func(args)

diff --git a/sodetlib/operations/uxm_relock.py b/sodetlib/operations/uxm_relock.py
@@ -260,7 +260,7 @@ def plot_channel_resonance(S, cfg, band, chan):
 
 @sdl.set_action()
 def uxm_relock(
-    S: SmurfControl, cfg, bands=None, show_plots=False,
+    S, cfg, bands=None, show_plots=False,
     setup_notches=False, new_master_assignment=False, reset_rate_khz=None,
     nphi0=None, skip_setup_amps=False):
     """