Merge branch 'development_py36' into stable_py36

.github/workflows/flow.yml

@@ -40,7 +40,7 @@ jobs:
     timeout-minutes: 40
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install dependencies
         run: |

.github/workflows/flow_amqp_consumer.yml

@@ -37,7 +37,7 @@ jobs:
     timeout-minutes: 40
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install dependencies
       # 2023-11-13 RS Added SSH config changes to see if it makes the tests more reliable

.github/workflows/flow_basic.yml

@@ -34,7 +34,7 @@ jobs:
     timeout-minutes: 20
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install dependencies
         run: |

.github/workflows/flow_mqtt.yml

@@ -37,7 +37,7 @@ jobs:
     timeout-minutes: 45
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install dependencies
         run: |

.github/workflows/flow_redis.yml

@@ -6,6 +6,7 @@ on:
   push:
     branches:
       - v03_disabled
+      - '*redis*'
 
     paths-ignore:
       - '.github/**'
@@ -37,7 +38,7 @@ jobs:
     timeout-minutes: 40
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       # Enable tmate debugging of manually-triggered workflows if the input option was provided
       # https://github.com/marketplace/actions/debugging-with-tmate

.github/workflows/ghcr-base.yml

@@ -26,7 +26,7 @@ jobs:
 
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1

.github/workflows/ghcr-dev.yml

@@ -27,7 +27,7 @@ jobs:
 
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1

.github/workflows/ghcr.yml

@@ -25,7 +25,7 @@ jobs:
 
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1

.github/workflows/publish-to-pypi.yml

@@ -12,7 +12,7 @@ jobs:
 
     steps:
       - name: Checkout source
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v3

debian/changelog

@@ -1,3 +1,32 @@
+metpx-sr3 (3.00.56) unstable; urgency=medium
+
+  * fix related to #1275 so "attempts 0" works for outages.
+  * fix #1278 regression: ack failure loops forever.
+  * fix #1271 AM charset issues, made some plugins more flexible.
+  * fix #1266 complain when explicitly asked to start disabled config.
+  * fix #1261 http performance regression.
+  * fix #1270 invalid mirroring of relative symlinks when realpath is
+    symlinked.  (requires next metpx-sr3c version 3.24.10rc2 ?3?)
+  * fix #1267 queueName before subtopic from this point forward.
+  * fix #1262 mtime referenced before assignment (download check fail.)
+  * fix #1271 renamer for AM socket generalized.
+  * fix #1255 remove malformed useless error message.
+  * incrementing github action version because of plaintive supplications
+  * demoted false-alarm error message about missing subscriptions.json to debug.
+  * NEW option *recursion* option ( #1237 ) (actually restored, was removed.)
+  * fix #1156 - NEW option *logDuplicates* 
+  * NEW EXPERIMENTAL option: *logJson*
+  * fix #1223 Switch from global QoS to per-consumer.
+  * AM sender problems identified in satnet testing.
+  * working on #35 augmenting .qname with subscriptions.json state file.
+  * working on #1183, sr3 manages instances > 100 correctly.
+  * PR #1228 Unquote secret_access_key in s3CloudSender plugin
+  * PR #1230 ... reduced spurious "unknown options" messages from config.
+  * fix #1199 - directory /  not interpreted properly.
+  * fix #1246 delete source when no options given
+
+ -- SSC-5CD2310S60 <[email protected]>  Thu, 31 Oct 2024 12:25:14 -0400
+
 metpx-sr3 (3.00.55post1) unstable; urgency=medium
 
   * fix #1242 sftp creates directories when it should not.

docs/source/Contribution/Release.rst

@@ -19,7 +19,7 @@ To publish a pre-release one needs to:
   * git pull
   * git checkout development_py36
   * git pull
-  * git merge development
+  * git merge --strategy-option=theirs development
 
 - validate that the correct version of C stack will be running when running flow tests.
   on each server::
@@ -84,15 +84,15 @@ To publish a pre-release one needs to:
      * git pull 
      * git checkout pre_release_py36
      * git pull
-     * git merge development_py36
+     * git merge --strategy-option=theirs development_py36
      * git push
      * git push origin o3.xx.yyrcz
 
      * git checkout development
      * git tag -a v3.xx.yy.rcZ -m "pre-release v3.xx.yy.rcz"
      * git checkout pre_release
      * git pull
-     * git merge development
+     * git merge --strategy-option=theirs development
      * git push
      * git push origin v3.xx.yyrcz
 
@@ -172,7 +172,7 @@ the stable release does not require any explicit testing.
      git pull
      git checkout stable
      git pull
-     git merge pre-release
+     git merge --strategy-option=theirs pre-release
      git push
 
      # there will be conflicts here for debian/changelog and sarracenia/_version.py
@@ -191,7 +191,7 @@ the stable release does not require any explicit testing.
      git pull
      git checkout stable_py36
      git pull
-     git merge pre_release_py36
+     git merge --strategy-option=theirs pre_release_py36
      git push
      # same editing required as above.
      git tag -a o3.xx.yy -m "o3.xx.yy"
@@ -256,6 +256,26 @@ ubuntu 18 is not compatible with the current pypi.org.
   * attach redhat 9 rpm 
   * attach windows exe ... see: `Building a Windows Installer`_
 
+Post-Release 
+------------
+
+Sometimes there is *just one bug* that should really be addressed in a stable release.
+Usually build with:
+
+* choose the branch (either pre-release or stable or similar for v2)
+* one (or more) *git cherry-pick* emergency fixes from some other branch.
+* one individual commit for sarracenia/_version.py to change add the post1 branch,
+* one commit to debian/changelog to update that with information about post1 and the changes therein
+
+These changes to debian/changelog and sarracenia/_version that will cause conflicts with 
+future development.  Afterwards, one needs to cherry-pick the debian/changelog back into 
+development to have a record of it.  This will likely cause a conflict, but it's easy to
+resolve.
+
+That's why a lot of the merges have *--strategy-option=theirs* because these postX releases
+cause conflicts.  
+
+
 Details
 -------
 
@@ -376,7 +396,7 @@ A tag should be created to identify the end of the cycle::
 Once the tag is in the development branch, promote it to stable::
 
    git checkout pre-release
-   git merge development
+   git merge --strategy-option=theirs development
    git push
 
 Once stable is updated on github, the docker images will be automatically upgraded, but
@@ -527,6 +547,12 @@ Repositories:
   * metpx-sr3-old -- on demand build sr3 packages from *stable_py36* branch.
   * metpx-sarracenia-release -- on deman build v2 packages from *v2_stable* branch.
 
+* Post-Release ( version.post1, post2 etc...)
+  usually the result of a cherry-pick for an emergency fix, and a commit or two about
+  debian/changelog and sarracenia/_version that will cause conflicts with future development.
+  * metpx-sr3 is the repository to target (using *stable* (or *v2_stable*) branch)
+
+
 for more discussion see `Which Version is stable <https://github.com/MetPX/sarracenia/issues/139>`_
 
 

docs/source/Explanation/CommandLineGuide.rst

@@ -542,6 +542,7 @@ The second row of output gives detailed headings within each category:
 The configurations are listed on the left. For each configuraion, the state
 will be:
 
+* cpuS:  process is expensive in CPU usage (runStateThreshold_cpuSlow)
 * disa:  disabled, configured not to run.
 * hung:  processes appear hung, not writing anything to logs.
 * idle:  all processes running, but no data or message transfers for too long (runStateThreshold_idle)
@@ -552,6 +553,8 @@ will be:
 * run:   all processes are running (and transferring, and not behind, and not slow... normal state.)
 * slow:  transfering less than minimum bytes/second ( runStateThreshold_slow )
 * stop:  no processes are running. 
+* stby:  Standby mode: all processes running, but messages are being stored in the local download_retry queue.
+* wVip:  process doesn't have the vip (only applies when the vip option is specified in the config)
 
 The next columns to the right give more information, detailing how many processes are Running, out of the number expected.
 For example, 3/3 means 3 processes or instances found of the 3 expected to be found.
@@ -2055,6 +2058,11 @@ are errors. The back-off can accumulate to the point where retries could be sepa
 or two. Once the server begins responding normally again, the programs will return to normal
 processing speed.
 
+If a failure will last for a while, one can stop the flow, configure *attempts 0* to fill the 
+retry queue without making vain attempts to download or send. At the end of the outage, return 
+*attempts* to normal, and the retry queue will gradually be drained when there
+is room in the current data flow. 
+
 
 EXAMPLES
 ========

docs/source/Explanation/DeploymentConsiderations.rst

@@ -147,6 +147,47 @@ Security Considerations
 This section is meant to provide insight to those who need to perform a security review
 of the application prior to implementation.
 
+
+Architecture
+~~~~~~~~~~~~
+
+Sarracenia can be a component in many solutions, and can be deployed as a cloud component.
+However, in it's rawest, simplest form, Sarracenia is not used like cloud services, where 
+one service is accessible from anywhere. It is more of a component or toolkit that is 
+expected to work with traditional network security zoning. Rather than have one service 
+for all, and requiring traffic/firewall exceptions and external security scanning to 
+intercept traffic, one deploys pump at each network zone demarcation. 
+
+Data is delivered to the pump at the demarcation point, and then another pump
+forwards data to the next zone. As part of demarcation processing, one can download a 
+file, run it through processing, such as malware scanning, and then only announce 
+it's availability to the following pump if it's ok.
+
+Each pump has independent authentication, and pump administrators 
+and users can define what traffic is made available to users on the other side of 
+the demarcation point. Pumps are chained together by copying from one to the next 
+to the next, where each one can have different access, purpose, and ownership.
+
+No formal federation or whole network identity is needed to pass data around 
+the network. Instead, each pump establishes authentication for the neigbouring pump. 
+If countries operated data pumps, one could imagine a situation like the following:
+The Russians and Americans want to transfer data but do not want to be exposed to each
+others' servers directly. The Russians could share with Kazakstan, The Kazakhs exchange
+with Korea, and Korea exchanges with Canada. The Americans only need to have
+a good relationship with the Canadians or Koreans. Each link in the chain 
+exposing themselves directly only to peers they have an explicit and 
+agreed relationship with. Each link in the chain can perform their own
+scanning and processing before accepting the data.
+
+.. image:: Concepts/sr3_flow_example.svg
+    :scale: 100%
+    :align: center
+
+In this example, you can see that there are the ddsr pumps deployed on internal
+operations zones, and they push or pull from pumps in other zones, such as another
+operations zone, or a public access zone. Pumps are expected to 
+mediate traffic travelling between network zones.
+
 Client
 ~~~~~~
 

docs/source/How2Guides/UPGRADING.rst

@@ -39,6 +39,25 @@ Installation Instructions
 git
 ---
 
+3.0.56
+------
+
+*CHANGE*: code refactor *sarracenia.credentials...* classes are now 
+*sarracenia.config.credentials* any code using credentials need to be
+updated.
+
+*CHANGE*: queue settings stored in subscriptions.json state file,
+instead of a .qname file, along with more information. Transition
+is perhaps complex. This version will read and write both files,
+so as to preserve ability to downgrade. later version will drop
+support for qname files.
+
+*CHANGE*: in configuration files: *subtopic* must come after
+the relevant queue naming options (queueName, queueShare)
+in prior releases, the queue naming was a global setting.
+In a future version, one will be able to subscribe to multiple
+queues with a single subscriber.
+
 3.0.54
 ------