IWXXM 2021-2 release (#26)

* Mostly due to changes of the 2021-2 schema release.
	TAF.py - minor enhancement to process <12 hr TAFs
        common/Encoder.py - changes due to processing <12 hr TAFs
        common/xmlConfig.py - update release variables from 3.0 to 2021-2
        swaEncoder.py - insert xsi:nil = true logic
        tcaEncoder.py - adding TC intensity change element,
                        revised logic for missing forecast maxWindSpeed and position,
                        insert xsi:nil = true logic
        vaaEncoder.py - insert xsi:nil = true logic

* Added new assertions for TCA product. Removed hardwired IWXXM URI in all tests.

* Updated README text and demo1.py script to include <12hr TAFs

README.md

@@ -8,19 +8,21 @@ This repository is a scientific product and is not official communication of the
 
 -------------------------------------------------------------------------------
 # Generate IWXXM From TAC
-This repository hosts software provided by the United States National Weather Service's [Meteorological Development Laboratory](https://vlab.ncep.noaa.gov/web/mdl) (MDL) that transforms Annex 3 Traditional Alphanumeric Code (TAC) forms into IWXXM v3.0 format.
+This repository hosts software provided by the United States National Weather Service's [Meteorological Development Laboratory](https://vlab.ncep.noaa.gov/web/mdl) (MDL) that transforms Annex 3 Traditional Alphanumeric Code (TAC) forms into IWXXM format.
 
 The ICAO Meteorological Information Exchange Model (IWXXM) is a format for reporting weather information in eXtensible Markup Language (XML). The IWXXM XML schemas, developed and hosted by the WMO in association with ICAO, are used to encode aviation products described in the Meteorological Service for International Air Navigation, Annex 3 to the Convention on International Civil Aviation.
 
-Version 3.0 of the IWXXM XML schemas encode METAR, SPECI, TAF, SIGMET, AIRMET, Volcanic Ash Advisory, Tropical Cyclone Advisory, and Space Weather Advisory reports.
+The IWXXM XML schemas encode METAR, SPECI, TAF, SIGMET, AIRMET, Volcanic Ash Advisory, Tropical Cyclone Advisory, and Space Weather Advisory reports and Significant Weather (SIGWX).
 
-This repository contains software, written exclusively in the Python language, that transforms the current TAC form of these reports into IWXXM XML documents. The advantage of the Python language is its popularity, rich functionality, and wide availability under many different computer operating systems.
+This repository contains software, written exclusively in the Python language, that transforms the data in the current TAC form of these reports into IWXXM XML documents. The advantage of the Python language is its popularity, rich functionality, and wide availability under many different computer operating systems.
 
 ## Introduction
-IWXXM v3.0 will become a WMO standard on 5 November 2020. Met Watch Offices shall disseminate METAR, SPECI, TAF, AIRMET, SIGMET products and Tropical Cyclone, Volcanic Ash and Space Weather Advisories in IWXXM form on that date.
+IWXXM became a ICAO standard on 5 November 2020. Various Met Watch Offices shall disseminate METAR, SPECI, TAF, AIRMET, SIGMET products and Tropical Cyclone, Volcanic Ash and Space Weather Advisories in IWXXM form after that date.
 
 As XML, and creating XML documents, may be unfamiliar technology to those without an IT background, MDL is providing software to assist those in creating the new XML documents based on IWXXM v3.0 schemas.
 
+It should be understood that the software provided here is a short-term solution as TAC forms of these products will cease to be a standard and no longer disseminated by 2029.
+
 ## Prequisites
 This software is written entirely in the Python language. Python interpreter version 2.7 or better is required.
 
@@ -90,7 +92,7 @@ The Encoder class requires that the input file contain one WMO AHL line, appropr
     S(A|P)[A-Z][A-Z]\d\d\s+[A-Z]{4}\s+\d{6}(\s+[ACR]{2}[A-Z])? # for METAR/SPECI
     FN\w\w\d\d\s+[A-Z]{4}\s+\d{6}(\s+[ACR]{2}[A-Z])? # Space Weather Advisories
     FK\w\w\d\d\s+[A-Z]{4}\s+\d{6}(\s+[ACR]{2}[A-Z])? # Tropical Cyclone Advisory
-    FT\w\w\d\d\s+[A-Z]{4}\s+\d{6}(\s+[ACR]{2}[A-Z])? # TAF
+    F(C|T)\w\w\d\d\s+[A-Z]{4}\s+\d{6}(\s+[ACR]{2}[A-Z])? # TAF
     FV\w\w\d\d\s+[A-Z]{4}\s+\d{6}(\s+[ACR]{2}[A-Z])? # Volcanic Ash Advisory
 And for capturing the individual TAC forms:
 

demo/demo1.py

@@ -103,7 +103,7 @@ def __init__(self):
                               gifts.METAR.Encoder(aerodromes)))
         self.encoders.append((re.compile(r'^FN[A-Z][A-Z]\d\d\s+[A-Z]{4}\s+\d{6}', re.MULTILINE),
                               gifts.SWA.Encoder()))
-        self.encoders.append((re.compile(r'^FT[A-Z][A-Z]\d\d\s+[A-Z]{4}\s+\d{6}', re.MULTILINE),
+        self.encoders.append((re.compile(r'^F(C|T)[A-Z][A-Z]\d\d\s+[A-Z]{4}\s+\d{6}', re.MULTILINE),
                               gifts.TAF.Encoder(aerodromes)))
         self.encoders.append((re.compile(r'FK\w\w\d\d\s+[A-Z]{4}\s+\d{6}', re.MULTILINE), gifts.TCA.Encoder()))
         self.encoders.append((re.compile(r'FV\w\w\d\d\s+[A-Z]{4}\s+\d{6}', re.MULTILINE), gifts.VAA.Encoder()))

gifts/TAF.py

@@ -24,9 +24,9 @@ def __init__(self, geoLocationsDB):
 
         super(Encoder, self).__init__()
 
-        self.re_AHL = re.compile(r'FT(?P<aaii>\w\w\d\d)\s+(?P<cccc>\w{4})\s+(?P<yygg>\d{6})(\s+(?P<bbb>[ACR]{2}[A-Z]))?')  # noqa:501
+        self.re_AHL = re.compile(r'F(?P<aaii>(C|T)\w\w\d\d)\s+(?P<cccc>\w{4})\s+(?P<yygg>\d{6})(\s+(?P<bbb>[ACR]{2}[A-Z]))?')  # noqa:501
         self.re_TAC = re.compile(r'^TAF(?:\s+(?:AMD|COR|CC[A-Z]|RTD))?\s+[A-Z]{4}.+?=', (re.MULTILINE | re.DOTALL))
-        self.T1T2 = "LT"
+        self.T1T2 = "L"
 
         self._Logger = logging.getLogger(__name__)
         self.decoder = tafDecoder.Decoder()

gifts/common/Encoder.py

@@ -45,11 +45,11 @@ def encode(self, text, receiptTime=None, **attrs):
                         decodedTAC['translatedBulletinReceptionTime'] = decodedTAC['translationTime']
 
                 elif 'err_msg' in decodedTAC:
-                    if self.T1T2 == 'L' or self.T1T2 == 'LT':
+                    if self.T1T2 == 'L':
                         try:
                             self._Logger.warning('Will not create IWXXM document for %s' % decodedTAC['ident']['str'])
                         except KeyError:
-                            self._Logger.warning('Bad observation, could not determine ICAO ID: %s' % tac)
+                            self._Logger.warning('Bad observation or TAF: Could not determine ICAO ID: %s' % tac)
                     else:
                         self._Logger.warning('Will not create IWXXM advisory because of a decoding error.')
 

gifts/common/xmlConfig.py

@@ -33,11 +33,11 @@
 # -----------------------------------------------------------------------------------
 #
 # IWXXM versioning
-_iwxxm = '3.0'
-_release = '3.0'
+_iwxxm = '2021-2'
+_release = '2021-2'
 #
 IWXXM_URI = 'http://icao.int/iwxxm/%s' % _iwxxm
-IWXXM_URL = 'http://schemas.wmo.int/iwxxm/%s/iwxxm.xsd' % _release
+IWXXM_URL = 'https://schemas.wmo.int/iwxxm/%s/iwxxm.xsd' % _release
 #
 # Path to file containing codes obtained from WMO Code Registry in RDF/XML format.
 #

gifts/swaEncoder.py

@@ -312,6 +312,8 @@ def postContent(self):
         indent = ET.SubElement(self.XMLDocument, 'remarks')
         if self.decodedTAC['remarks'] == 'NIL':
             indent.set('nilReason', self.codes[des.NIL][des.NA][0])
+            indent.set('xsi:nil', 'true')
+
         else:
             indent.text = self.decodedTAC['remarks']
 

gifts/tcaEncoder.py

@@ -164,21 +164,22 @@ def result(self, parent, token, fhr):
         if fhr == '0':
             self.doObservedConditions(ET.SubElement(parent, 'observation'), token)
         else:
-            indent = ET.SubElement(parent, 'forecast')
-            if 'windSpeed' in token:
 
-                indent1 = ET.Element('TropicalCycloneForecastConditions')
-                indent1.set('gml:id', deu.getUUID())
-                self.itime(indent1, token['dtg'])
-                self.cyclonePosition(indent1, token)
+            indent = ET.SubElement(parent, 'forecast')
+            indent1 = ET.SubElement(indent, 'TropicalCycloneForecastConditions')
+            indent1.set('gml:id', deu.getUUID())
+            self.itime(indent1, token['dtg'])
+            self.cyclonePosition(indent1, token)
 
+            try:
                 indent2 = ET.SubElement(indent1, 'maximumSurfaceWindSpeed')
-                indent2.set('uom', token['windSpeed']['uom'])
                 indent2.text = token['windSpeed']['value']
-                indent.append(indent1)
+                indent2.set('uom', token['windSpeed']['uom'])
 
-            else:
-                indent.set('nilReason', self.codes[des.NIL][des.NOOPRSIG][0])
+            except KeyError:
+                indent2.set('uom', 'N/A')
+                indent2.set('nilReason', self.codes[des.NIL][des.NOOPRSIG][0])
+                indent2.set('xsi:nil', 'true')
 
     def itime(self, parent, dtg):
 
@@ -191,18 +192,19 @@ def itime(self, parent, dtg):
     def cyclonePosition(self, parent, token):
 
         indent = ET.SubElement(parent, 'tropicalCyclonePosition')
-        if 'position' in token:
-
-            indent1 = ET.SubElement(indent, 'gml:Point')
-            indent1.set('gml:id', deu.getUUID())
-            indent1.set('axisLabels', des.axisLabels)
-            indent1.set('srsName', des.srsName)
-            indent1.set('srsDimension', des.srsDimension)
-            indent2 = ET.SubElement(indent1, 'gml:pos')
+        indent1 = ET.Element('gml:Point')
+        indent1.set('gml:id', deu.getUUID())
+        indent1.set('axisLabels', des.axisLabels)
+        indent1.set('srsName', des.srsName)
+        indent1.set('srsDimension', des.srsDimension)
+        indent2 = ET.SubElement(indent1, 'gml:pos')
+        try:
             indent2.text = token['position']
+            indent.append(indent1)
 
-        else:
-            indent.set('nilReason', self.codes[des.NIL][des.MSSG][0])
+        except KeyError:
+            indent.set('nilReason', self.codes[des.NIL][des.NA][0])
+            indent.set('xsi:nil', 'true')
 
     def doObservedConditions(self, parent, token):
 
@@ -228,6 +230,9 @@ def doObservedConditions(self, parent, token):
             indent1.text = token['movement']['spd']
             indent1.set('uom', token['movement']['uom'])
 
+        indent1 = ET.SubElement(indent, 'intensityChange')
+        indent1.text = {'INTSF': 'INTENSIFY', 'WKN': 'WEAKEN'}.get(self.decodedTAC['intstChange'], 'NO_CHANGE')
+
         indent1 = ET.SubElement(indent, 'centralPressure')
         indent1.text = self.decodedTAC['minimumPressure']['value']
         indent1.set('uom', self.decodedTAC['minimumPressure']['uom'])
@@ -307,6 +312,8 @@ def postContent(self):
         indent = ET.SubElement(self.XMLDocument, 'remarks')
         if self.decodedTAC['remarks'] == 'NIL':
             indent.set('nilReason', self.codes[des.NIL][des.NA][0])
+            indent.set('xsi:nil', 'true')
+
         else:
             indent.text = self.decodedTAC['remarks']
 

gifts/vaaEncoder.py

@@ -115,8 +115,10 @@ def preamble(self):
         child = ET.SubElement(self.XMLDocument, 'stateOrRegion')
         if 'UNKNOWN' not in self.decodedTAC['region']:
             child.text = self.decodedTAC['region']
+
         else:
             child.set('nilReason', self.codes[des.NIL][des.UNKNWN][0])
+            child.set('xsi:nil', 'true')
 
         child = ET.SubElement(self.XMLDocument, 'summitElevation')
         if self.decodedTAC['summit']['elevation'].isdigit():
@@ -125,8 +127,10 @@ def preamble(self):
 
         elif 'SFC' in self.decodedTAC['summit']['elevation']:
             child.set('nilReason', self.codes[des.NIL][des.NA][0])
+            child.set('xsi:nil', 'true')
         else:
             child.set('nilReason', self.codes[des.NIL][des.UNKNWN][0])
+            child.set('xsi:nil', 'true')
 
         child = ET.SubElement(self.XMLDocument, 'advisoryNumber')
         child.text = self.decodedTAC['advisoryNumber']
@@ -138,10 +142,16 @@ def preamble(self):
             child = ET.Element('colourCode')
             if 'GIVEN' in self.decodedTAC['colourCode']:
                 child.set('nilReason', self.codes[des.NIL][des.WTHLD][0])
+                child.set('xsi:nil', 'true')
+
             elif self.decodedTAC['colourCode'] == 'UNKNOWN':
                 child.set('nilReason', self.codes[des.NIL][des.UNKNWN][0])
+                child.set('xsi:nil', 'true')
+
             elif self.decodedTAC['colourCode'] == 'NIL':
                 child.set('nilReason', self.codes[des.NIL][des.MSSG][0])
+                child.set('xsi:nil', 'true')
+
             else:
                 child.set('xlink:href', self.codes[des.COLOUR_CODES][self.decodedTAC['colourCode']][0])
 
@@ -153,6 +163,8 @@ def preamble(self):
         child = ET.SubElement(self.XMLDocument, 'eruptionDetails')
         if 'UNKNOWN' in self.decodedTAC['details']:
             child.set('nilReason', self.codes[des.NIL][des.UNKNWN][0])
+            child.set('xsi:nil', 'true')
+
         else:
             child.text = self.decodedTAC['details']
 
@@ -202,6 +214,8 @@ def volcano(self, parent):
         indent2 = ET.SubElement(indent1, 'position')
         if 'UNKNOWN' in self.decodedTAC['volcanoLocation']:
             indent2.set('nilReason', self.codes[des.NIL][des.UNKNWN][0])
+            indent2.set('xsi:nil', 'true')
+
         else:
             indent3 = ET.SubElement(indent2, 'gml:Point')
             indent3.set('axisLabels', des.axisLabels)
@@ -394,8 +408,10 @@ def airspaceVolume(self, parent, lyr):
     def postContent(self):
 
         indent = ET.SubElement(self.XMLDocument, 'remarks')
-        if self.decodedTAC['remarks']  == 'NIL':
+        if self.decodedTAC['remarks'] == 'NIL':
             indent.set('nilReason', self.codes[des.NIL][des.NA][0])
+            indent.set('xsi:nil', 'true')
+
         else:
             indent.text = self.decodedTAC['remarks']