forked from open-power/vpdtools
-
Notifications
You must be signed in to change notification settings - Fork 0
/
createVpd.py
executable file
·1264 lines (1070 loc) · 55.4 KB
/
createVpd.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python
# Created 01/26/15 by Jason Albert
# Program to create VPD images from input template files
# IBM_PROLOG_BEGIN_TAG
# This is an automatically generated prolog.
#
# OpenPOWER HostBoot Project
#
# Contributors Listed Below - COPYRIGHT 2010,2014
# [+] International Business Machines Corp.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
#
# IBM_PROLOG_END_TAG
############################################################
# Imports - Imports - Imports - Imports - Imports - Imports
############################################################
import os
# Get the path the script resides in
scriptPath = os.path.dirname(os.path.realpath(__file__))
import sys
sys.path.insert(0,scriptPath + "/pymod");
import out
import xml.etree.ElementTree as ET
import struct
import binascii
import re
import argparse
import textwrap
import os
# Define basestring for python3 compatibility
if not hasattr(__builtins__, "basestring"): basestring = (str, bytes)
############################################################
# Classes - Classes - Classes - Classes - Classes - Classes
############################################################
# This parser extension is necessary to save comments and write them back out in the final file
# By default, element tree doesn't preserve comments
# https://stackoverflow.com/questions/33573807/faithfully-preserve-comments-in-parsed-xml-python-2-7/
class CommentedTreeBuilder(ET.TreeBuilder):
def __init__(self, *args, **kwargs):
super(CommentedTreeBuilder, self).__init__(*args, **kwargs)
def comment(self, data):
self.start(ET.Comment, {})
self.data(data)
self.end(ET.Comment)
class RecordInfo:
"""Stores the info about each vpd record"""
def __init__(self):
# The packed record in memory
self.record = bytearray()
# The packed ecc in memory
self.ecc = bytearray()
# The name of the record where the toc entry is located
self.tocName = None
# The location of the Record Offset in toc record
self.tocRecordOffset = None
# The location of the Record Length in toc record
self.tocRecordLength = None
# The location of the ECC Offset in toc record
self.tocEccOffset = None
# The location of the ECC Length in toc record
self.tocEccLength = None
############################################################
# Function - Functions - Functions - Functions - Functions
############################################################
# Find file in a given path or paths
# searchPath comes from the -i/--inpath option
def findFile(filename, searchPath):
found = False
paths = searchPath.split(os.path.pathsep)
for path in paths:
if os.path.exists(os.path.join(path, filename)):
found = 1
break
if found:
return os.path.abspath(os.path.join(path, filename))
else:
return None
# Parses a vpd xml file using ET. ET will generate errors for bad xml syntax
# Actual checking/validation of the xml contents will be done elsewhere
def parseXml(xmlFile):
errorsFound = 0
# Get the full path to the file given
fullPathFile = findFile(xmlFile, clInputPath)
if (fullPathFile == None):
out.error("The xml file %s could not be found!" % (xmlFile))
# If we can't open the file, we can't do anything else in this function. Bail..
return (1, None)
# Let the user know what file we are reading
out.msg("Parsing file %s" % fullPathFile)
# Read in the file with ET
# If there are tag mismatch errors or other general gross format problems, it will get caught here
# Once we return from this function, then we'll check to make sure only supported tags were given, etc..
# Invoke the extended comment parser, which will handle preserving comments in the output file
parser = ET.XMLParser(target=CommentedTreeBuilder())
try:
root = ET.parse(fullPathFile, parser=parser).getroot()
except Exception as e:
out.error("Unable to parse %s!" % fullPathFile)
out.error("Check your file for basic XML formatting issues, or missing toplevel <vpd> tag")
out.error("Python Exception: %s" % e)
return (1, None)
# Print the top level tags from the parsing
if (clDebug):
out.debug("Top level tag/attrib found")
for child in root:
out.debug("%s %s" % (child.tag, child.attrib))
# Vary our return based upon the errorsFound
if (errorsFound):
return (errorsFound, None)
else:
return(0, root)
# Function to write out the resultant xml file
def writeXml(manifest, outputFile):
tree = ET.ElementTree(manifest)
tree.write(outputFile, encoding="utf-8", xml_declaration=True)
# Now rip it through xmllint quick to cleanup formatting problems from the ET print
if (os.path.isfile("/usr/bin/xmllint")):
rc = os.system("/usr/bin/xmllint --format %s -o %s" % (outputFile, outputFile))
if (rc):
out.error("Unable to call xmllint to fix xml formatting")
return(rc)
else:
out.warn("xmllint not installed - no formatting cleanup done!")
return None
# Check the <vpd> XML to make sure the required elements are found
def checkElementsVpd(root):
errorsFound = 0
# Do some basic error checking of what we've read in
# Make sure the root starts with the vpd tag
# If it doesn't, it's not worth syntax checking any further
# This is the only time we'll just bail instead of accumulating
if (root.tag != "vpd"):
out.error("%s does not start with a <vpd> tag. No further checking will be done until fixed!" % tvpdFile)
exit(1)
# We at least have a proper top level vpd tag, so loop thru the rest of the levels and check for any unknown tags
# This will also be a good place to check for the required tags
# Define the expected tags at this level
vpdTags = {"name" : 0, "size" : 0, "VD" : 0, "record" : 0}
# Go thru the children at this level
for child in root:
# Comments aren't basestring tags
if not isinstance(child.tag, basestring):
continue
# See if this is a tag we even expect
if child.tag not in vpdTags:
out.error("Unsupported tag <%s> found while parsing the <vpd> level" % child.tag)
errorsFound += 1
# It was a supported tag
else:
vpdTags[child.tag] += 1
# Done looping through tags, do checking of what we found at the vpd level
# These tags are not required in record only mode
if (not clRecordMode):
for tag in ["name", "size", "VD"]:
if (vpdTags[tag] != 1):
out.error("The tag <%s> was expected to have a count of 1, but was found with a count of %d" %
(tag, vpdTags[tag]))
errorsFound += 1
# Make sure at least one record tag was found
if (vpdTags["record"] == 0):
out.error("At least one <record> must be defined for the file to be valid!")
errorsFound += 1
# In record only mode, make sure there is only 1 record in the file
if (clRecordMode and vpdTags["record"] != 1):
out.error("Only one <record> definition per file is supported in record mode")
out.error("The number of <record> definitions found in your file: %d" % vpdTags["record"])
errorsFound += 1
return errorsFound
# Check the <record> XML to make sure the required elements are found
def checkElementsRecord(record):
errorsFound = 0
# Define the expected tags at this level
recordTags = {"rdesc" : 0, "keyword" : 0, "rtvpdfile" : 0, "rbinfile" : 0}
# Make sure the record has a name attrib, save for later use
recordName = record.attrib.get("name")
if (recordName == None):
out.error("A <record> tag is missing the name attribute")
errorsFound += 1
recordName = "INVALID" # Set the invalid name so the code below can use it without issue
# Loop thru the tags defined for this record
for child in record:
# Comments aren't basestring tags
if not isinstance(child.tag, basestring):
continue
# See if this is a tag we even expect
if child.tag not in recordTags:
out.error("Unsupported tag <%s> found while parsing the <record> level for record %s" % (child.tag, recordName))
errorsFound += 1
# It was a supported tag
else:
recordTags[child.tag] += 1
# Done looping through tags
# We've checked for unknown record tags, now make sure we've got the right number, they don't conflict, etc..
recordTagTotal = bool(recordTags["keyword"]) + bool(recordTags["rbinfile"]) + bool(recordTags["rtvpdfile"])
# keyword, rbinfile and rtvpdfile are mutually exclusive. Make sure we have only one
if (recordTagTotal > 1):
out.error("For record %s, more than one tag of type keyword, rbinfile or rtvpdfile was given!" % (recordName))
out.error("Use of only 1 at a time is supported for a given record!")
errorsFound += 1
# We checked if we had more than 1, let's make sure we have at least 1
if (recordTagTotal < 1):
out.error("For record %s, 0 tags of type keyword, rbinfile or rtvpdfile were given!" % (recordName))
out.error("1 tag of the 3 must be in use for the record to be valid!")
errorsFound += 1
# Make sure the rdesc is available
if (recordTags["keyword"] and recordTags["rdesc"] != 1):
out.error("The tag <rdesc> was expected to have a count of 1, but was found with a count of %d for record %s" %
(recordTags["rdesc"], recordName))
errorsFound += 1
return (errorsFound, recordName)
# Check the <keyword> XML to make sure the required elements are found
def checkElementsKeyword(keyword, recordName):
errorsFound = 0
# Define the expected tags at this level
keywordTags = {"kwdesc" : 0, "kwformat" : 0, "kwlen" : 0, "kwdata" : 0, "ktvpdfile" : 0}
# Make sure the keyword has a name attrib, save for later use
keywordName = keyword.attrib.get("name")
if (keywordName == None):
out.error("<keyword> tag in record %s is missing the name attribute" % (recordName))
errorsFound += 1
keywordName = "INVALID" # Set the invalid name so the code below can use it without issue
# Loop thru the tags defined for this keyword
for child in keyword:
# Comments aren't basestring tags
if not isinstance(child.tag, basestring):
continue
# See if this is a tag we even expect
if child.tag not in keywordTags:
out.error("Unsupported tag <%s> found while parsing the <keyword> level for keyword %s in record %s" %
(child.tag, keywordName, recordName))
errorsFound += 1
# It was a supported tag
else:
keywordTags[child.tag] += 1
# Done looping through tags
# We've checked for unknown keyword tags, now make sure we have the right number of each
# The default is we expect the regular keyword tags to be there, so default to 1
keywordTagCount = 1
# If we found a ktvpdfile tag, make sure we only had one of them
if (keywordTags["ktvpdfile"] != 0):
if (keywordTags["ktvpdfile"] > 1):
out.error("The tag <ktvpdfile> is only allowed to be used once for keyword %s in record %s" %
(keywordName, recordName))
errorsFound += 1
# We had a ktvpdfile, now we don't want any of the regular keyword tags
keywordTagCount = 0
# Depending upon the state of ktvpdfile, check to ensure we are in the right state
for tag in ["kwdesc", "kwformat", "kwlen", "kwdata"]:
if (keywordTags[tag] != keywordTagCount):
out.error("The tag <%s> was expected to have a count of %d, but was found with a count of %d for keyword %s in record %s" %
(tag, keywordTagCount, keywordTags[tag], keywordName, recordName))
errorsFound += 1
return (errorsFound, keywordName)
# Function to write properly packed/encoded data to the vpdFile
def writeDataToVPD(vpdFile, data, offset = None):
rc = 0
# If the user gave an offset, save the current offset and seek to the new one
entryOffset = None
if (offset != None):
entryOffset = vpdFile.tell()
vpdFile.seekg(offset)
# Write the data
vpdFile.write(data)
# Restore the offset to original location if given
if (offset != None):
vpdFile.seekg(entryOffset)
return rc
# Turn the tvpd keyword data into packed binary data we can write to the file
def packKeyword(keyword, length, data, format):
# We'll return a bytearray of the packed data
keywordPack = bytearray()
# Write the keyword
keywordPack += bytearray(keyword.encode())
# Write the length
# The < at the front says to pack it little endian
if (keyword[0] == "#"): # Keywords that start with pound have a 2 byte length
# H is 2 bytes
keywordPack += struct.pack("<H", length)
else:
# B is 1 byte
keywordPack += struct.pack("<B", length)
# Write the data
# If the user didn't provide data = length given, we'll pad the end with 0's
if (format == "ascii"):
# Pad if necessary
data = data.ljust(length, '\0')
# Write it
keywordPack += bytearray(data.encode())
elif (format == "hex"):
# Remove white space and carriage returns from the data before we get to fromhex
# If we don't, it throws off the ljust logic below to set the field to proper length
data = data.replace(" ","")
data = data.replace("\n","")
# Pad if necessary (* 2 to convert nibble data to byte length)
data = data.ljust((length * 2), '0')
# Write it
keywordPack += bytearray.fromhex(data)
else:
out.error("Unknown format type %s passed into packKeyword" % format)
return None
# The keyword is packed, send it back
return keywordPack
# Calculate the length of the PF record
def calcPadFill(record):
pfLength = 0
# The PF keyword must exist
# The keyword section of record must be at least 40 bytes long, padfill will be used to achieve that
# If the keyword section is over over 40, it must be aligned on word boundaries and PF accomplishes that
# The record passed in at this point is the keywords + 3 other bytes (LR Tag & Record Length)
# Those 3 bytes happen to match the length of the PF keyword and its length which needs to be in the calculation
# So we'll just use the length of the record, but it's due to those offsetting lengths of 3
pfLength = 40 - len(record)
if (pfLength < 1):
# It's > 40, so now we just need to fill to nearest word
pfLength = (4 - (len(record) % 4))
return pfLength
# Check input hex data for proper formatting
def checkHexDataFormat(kwdata):
# Remove white space and carriage returns from the kwdata
kwdata = kwdata.replace(" ","")
kwdata = kwdata.replace("\n","")
# Now look to see if there are any characters other than 0-9 & a-f
match = re.search("([^0-9a-fA-F]+)", kwdata)
if (match):
out.error("A non hex character \"%s\" was found at %s in the kwdata" % (match.group(), match.span()))
return (match, kwdata)
############################################################
# Main - Main - Main - Main - Main - Main - Main - Main
############################################################
rc = 0
################################################
# Command line options
# Create the argparser object
# We disable auto help options here and add them manually below. This is so we can get all the optional args in 1 group
parser = argparse.ArgumentParser(description='The VPD image creation tool', add_help=False,
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=textwrap.dedent('''\
Examples:
./createVpd.py -m examples/simple/simple.tvpd -o /tmp
./createVpd.py -m examples/rbinfile/rbinfile.tvpd -i examples/rbinfile -o /tmp
'''))
# Create our group of required command line args
reqgroup = parser.add_argument_group('Required Arguments')
reqgroup.add_argument('-m', '--manifest', help='The input file detailing all the records and keywords to be in the image', required=True)
reqgroup.add_argument('-o', '--outpath', help='The output path for the files created by the tool', required=True)
# Create our group of optional command line args
optgroup = parser.add_argument_group('Optional Arguments')
optgroup.add_argument('-h', '--help', action="help", help="Show this help message and exit")
optgroup.add_argument('-d', '--debug', help="Enables debug printing", action="store_true")
optgroup.add_argument('-c', '--record-mode', help="The input is a record only file. No output VPD binary created.", action="store_true")
optgroup.add_argument('-r', '--binary-records', help="Create binary files for each record in the template", action="store_true")
optgroup.add_argument('-k', '--binary-keywords', help="Create binary files for each keyword in the template", action="store_true")
optgroup.add_argument('-i', '--inpath', help="The search path to use for the files referenced in the manifest")
# We've got everything we want loaded up, now look for it
args = parser.parse_args()
# Get the manifest file and get this party started
clManifestFile = args.manifest
# Look for output path
clOutputPath = args.outpath
# Make sure the path exists, we aren't going to create it
if (os.path.exists(clOutputPath) != True):
out.error("The given output path %s does not exist!" % clOutputPath)
out.error("Please create the output directory and run again")
exit(1)
# Look for input path
clInputPath = args.inpath
# Make sure the path exists
if (clInputPath != None):
# Add the CWD onto the path so the local directory is always looked at
clInputPath += os.pathsep + "."
else:
# Set it the CWD since it will be used throughout the program and having it set to None breaks things
clInputPath = "."
# Debug printing
clDebug = args.debug
# Record only mode
clRecordMode = args.record_mode
# Create separate binary files for each record
clBinaryRecords = args.binary_records
# Create separate binary files for each keyword
clBinaryKeywords = args.binary_keywords
# We are going to do this in 3 stages
# 1 - Read in the manifest and any other referenced files. This will create a complete XML description of the VPD
# We will also check to make sure that all required tags are given and no extra tags exist
# 2 - Parse thru the now complete vpd tree and make sure the data within the tags is valid.
# These are checks like data not greater than length, etc..
# 3 - With the XML and contents verified correct, loop thru it again and write out the VPD data
#
# Note: Looping thru the XML twice between stage 1 and 2 makes it easier to surface multiple errors to the user at once.
# If we were trying to both validate the xml and data at once, it would be harder to continue and gather multiple errors like we do now
################################################
# Work with the manifest
out.setIndent(0)
out.msg("==== Stage 1: Parsing VPD XML files")
out.setIndent(2)
# Accumulate errors and return the total at the end
# This allows the user to see all mistakes at once instead of iteratively running
errorsFound = 0
# Read in the top level manifest file and create the xml manifest tree
# If this parse gets an error, it's a hard stop since the rest of the code would do nothing
(rc, manifest) = parseXml(clManifestFile)
if (rc):
out.error("Please check your -m or -i cmdline options for typos")
exit(rc)
# We have the top level manifest tree. Make sure we have all the required elements in the <vpd> section
# Accumulate errors for this checking
errorsFound += checkElementsVpd(manifest)
# We've parsed and check the <vpd> section, now do the same to all <records> children
for record in manifest.iter("record"):
# Accumulate errors for this checking
# This returning non 0 would indicate a problem at the base record level the user will have to fix
# However, we'll still continue to try and parse any rtvpdfile and keyword entries contained in this record
# This is so we can expose as many errors to the user all at once
(rc, recordName) = checkElementsRecord(record)
errorsFound += rc
# See if a rtvpdfile was given and if so, load it in
rtvpdfile = record.find("rtvpdfile")
if (rtvpdfile != None):
# Read in the rtvpdfile
rtvpdfileName = rtvpdfile.text
(rc, recordTvpd) = parseXml(rtvpdfile.text)
if (rc):
out.error("The <rtvpdfile> given could not be found.")
errorsFound += 1
continue
# Early versions of these files could start with <vpd> tag, handle that by getting down a level to the record
# If it's already the top level entry, then just assign over
if (recordTvpd.tag == "vpd"):
newRecord = recordTvpd.find("record")
else:
newRecord = recordTvpd
# --------
# Check the contents read in from the rtvpdfile
(rc, newRecordName) = checkElementsRecord(newRecord)
errorsFound += rc
# --------
# Make sure the record found in rtvpdfile is the same as the record in the manifiest
# We have to do this error check here because the recordName doesn't exist in parseTvpd
if (newRecordName != recordName):
out.error("The record (%s) found in %s doesn't match the record name in the manifest (%s)" %
(newRecordName, rtvpdfile.text, recordName))
errorsFound += 1
continue
# Insert a comment with a name of the file the record came from
comment = ET.Comment(" Imported rtvpdfile contents - %s " % rtvpdfileName)
comment.tail = "\n"
newRecord.insert(0, comment)
else:
# It's not a rtvpdfile record. Set newRecord to record for all the remaining code below
# This is done so that any ktvpdfile references that are read in get merged into the containing record
# That containing record then gets merged into the main manifest in the 2nd merge below
# The end result is a manifest that contains no references to external files and can be second stage processed
newRecord = record
newRecordName = recordName
# Done handling the record level
# We can now loop through the keywords in the records and check them
# Look for ktvpdfile lines
for keyword in newRecord.iter("keyword"):
# Accumulate errors for this checking
# This returning non 0 would indicate a problem at the base keyword level the user will have to fix
# However, we'll still continue to try and parse any ktvpdfile entries contained in this record
# This is so we can expose as many errors to the user all at once
(rc, keywordName) = checkElementsKeyword(keyword, newRecordName)
errorsFound += rc
# Track if we hit the conditionals that cause a newKeyword replacement to be needed
newKeywordReplace = False
# See if a ktvpdfile was given and if so, load it in
ktvpdfile = keyword.find("ktvpdfile")
if (ktvpdfile != None):
# Read in the ktvpdfile
ktvpdfileName = ktvpdfile.text
(rc, newKeyword) = parseXml(ktvpdfile.text)
if (rc):
out.error("The <ktvpdfile> given could not be found.")
errorsFound += 1
continue
# --------
# Check the contents read in from the ktvpdfile
(rc, newKeywordName) = checkElementsKeyword(newKeyword, newRecordName)
errorsFound += rc
# --------
# Make sure the keyword found in ktvpdfile is the same as the keyword in the manifiest
# We have to do this error check here because the keywordName doesn't exist in parseTvpd
if (newKeywordName != keywordName):
out.error("The keyword (%s) found in %s doesn't match the keyword name in the manifest (%s)" %
(newKeywordName, ktvpdfile.text, keywordName))
errorsFound += 1
continue
# Insert a comment with a name of the file the record came from
comment = ET.Comment(" Imported ktvpdfile contents - %s " % ktvpdfileName)
comment.tail = "\n"
newKeyword.insert(0, comment)
# We were successful, make our replacement active
newKeywordReplace = True
# See if the kwformat is a binary file ("bin")
# If it is, load it in and turn it into a hex data keyword for the rest of the run
# This is necessary so when the output tvpd is written, we write out the actual data instead of a reference to the file
elif (keyword.find("kwformat").text == "bin"):
# Get the name of the file out of the kwdata
databinfileName = keyword.find("kwdata").text
# Check to make sure the file can be found
databinfile = findFile(databinfileName, clInputPath)
if (databinfile == None):
out.error("The input binary data file %s could not be found! Please check your tvpd or input path." % databinfileName)
errorsFound += 1
continue
# We were able to read the file in successfully
# - Create our newKeyword for the replacement
newKeyword = keyword
# - Set our data type
newKeyword.find("kwformat").text = "hex"
# - Read in our bin data & store it as hex ascii data
kwdata = open(databinfile, mode='rb').read()
newKeyword.find("kwdata").text = binascii.hexlify(kwdata)
# Insert a comment with a name of the file the data came from
comment = ET.Comment(" Imported bin contents of file as hex - %s " % databinfileName)
comment.tail = "\n"
newKeyword.insert(list(newKeyword).index(newKeyword.find("kwdata")), comment)
# We were successful, make our replacement active
newKeywordReplace = True
if (newKeywordReplace):
# Merge the new keyword into the record
# ET doesn't have a replace function. You can do an extend/remove, but that changes the order of the file
# The goal is to preserve order, so that method doesn't work
# The below code will insert the newKeyword in the list above the current matching keyword definition
# Then remove the original keyword definition, preserving order
newRecord.insert(list(newRecord).index(keyword), newKeyword)
newRecord.remove(keyword)
# End of the record loop
# Merge the new record into the main manifest
# ET doesn't have a replace function. You can do an extend/remove, but that changes the order of the file
# The goal is to preserve order, so that method doesn't work
# The below code will insert the newRecord in the list above the current matching record definition
# Then remove the original record definition, preserving order
manifest.insert(list(manifest).index(record), newRecord)
manifest.remove(record)
# All done with error checks, bailout if we hit something
if (errorsFound):
out.msg("")
out.error("%d error%s found in the xml. Please review the above errors and correct them." %
(errorsFound, "s" if (errorsFound > 1) else ""))
exit(errorsFound)
################################################
# Verify the tvpd XML
# read thru the complete tvpd and verify/check actual tag contents
out.setIndent(0)
out.msg("==== Stage 2: Verifying tvpd syntax")
out.setIndent(2)
errorsFound = 0
# Keep a dictionary of the record names we come across, will let us find duplicates
recordNames = dict()
# Do our top level <vpd> validation of tag contents
# Nothing to validate for the name, however grab it for use in later operations
# In normal mode, the user has to specify the output file name in the input
# For record only mode, we only use the input filename as the output file name
if (not clRecordMode):
vpdName = manifest.find("name").text
# If the user passed in the special name of FILENAME, we'll use in the input file name, minus the extension, as the output
if (vpdName == "FILENAME"):
vpdName = os.path.splitext(os.path.basename(clManifestFile))[0]
else:
vpdName = os.path.basename(clManifestFile)
# Validate the <size> is given in proper syntax
if (not clRecordMode):
vpdSize = manifest.find("size").text
# Make a new string with only the number
maxSizeBytes = re.match('[0-9]*', vpdSize).group()
# --------
# Check to see if the number is even there
if (maxSizeBytes == ''):
maxSizeBytes = '0'
out.error("No number detected in the size string. Format of string must be number first, then units, e.g. 16KB.")
out.error("Remove any characters or white space from in front of the number.")
errorsFound += 1
# --------
# Make a new string with the number removed
sizeUnits = vpdSize[len(maxSizeBytes):]
# Remove a space, if one was inserted between the number and units
whitespace = re.match(' *', sizeUnits).group()
sizeUnits = sizeUnits[len(whitespace):]
# Check the units to see if they are okay
if (sizeUnits.lower() == "kb"):
maxSizeBytes = int(maxSizeBytes) * 1024
elif (sizeUnits.lower() == "b"):
maxSizeBytes = int(maxSizeBytes)
elif (sizeUnits.lower() == "mb"):
maxSizeBytes = int(maxSizeBytes) * 1024 * 1024
elif (sizeUnits == ""):
out.error("Please specify units at the end of the size string. Acceptable units: B/KB/MB")
errorsFound += 1
else:
out.error("Unexpected units in the size string. Expected: B/KB/MB. Yours: %s" % sizeUnits)
errorsFound += 1
# Loop thru our records and then thru the keywords in each record
for record in manifest.iter("record"):
# Pull the record name out for use throughout
recordName = record.attrib.get("name")
out.msg("Verifying record %s" % recordName)
out.setIndent(4)
# --------
# Make sure we aren't finding a record we haven't already seen
if (recordName in recordNames):
out.error("The record \"%s\" has previously been defined in the tvpd" % recordName)
errorsFound += 1
else:
recordNames[recordName] = 1
# --------
# Make sure the record name is 4 charaters long
if (len(recordName) != 4):
out.error("The record name entry \"%s\" is not 4 characters long" % recordName)
errorsFound += 1
# --------
# Do very basic checking on the rbinfile if found
# It is assumed that this file was generated by this tool at an earlier date, so it should be format correct
# We'll simply ensure it is actually a record that goes with this record name
if (record.find("rbinfile") != None):
# Get the name
rbinfile = record.find("rbinfile").text
# Get the full path to the file given
rbinfile = findFile(rbinfile, clInputPath)
if (rbinfile == None):
out.error("The rbinfile %s could not be found! Please check your tvpd or input path" % (rbinfile))
errorsFound += 1
break
# It does, read it in so we can check the record name
out.msg("Reading rbinfile %s" % (rbinfile))
rbinfileContents = open(rbinfile, mode='rb').read()
# --------
# Check the record name
# This is just the hard coded offset into any record where the contents of the RT keyword would be found
if (recordName != rbinfileContents[6:10].decode()):
out.error("The record name found %s in %s, does not match the name of the record %s in the tvpd" %
(rbinfileContents[6:10].decode(), rbinfile, recordName))
clErrors+=1
# --------
# For the keyword tags we'll do much more extensive checking
if (record.find("keyword") != None):
# Track the keywords we come across so we can find duplicate
keywordNames = dict()
# Loop through the keywords and verify them
for keyword in record.iter("keyword"):
# Pull the keyword name out for use throughout
keywordName = keyword.attrib.get("name")
# Setup a dictionary of the supported tags
kwTags = {"keyword" : False, "kwdesc" : False, "kwformat" : False, "kwlen" : False, "kwdata" : False}
# Setup a dictionary of the supported tags in the kwdata tag
kwdTags = {"ascii" : False, "hex" : False}
# --------
# Make sure we aren't finding a record we haven't already seen
if (keywordName in keywordNames):
out.error("The keyword \"%s\" has previously been defined in record %s" % (keywordName, recordName))
errorsFound += 1
else:
keywordNames[keywordName] = 1
# --------
# We'll loop through all the tags found in this keyword and check for all required and any extra ones
for kw in keyword.iter():
# Comments aren't basestring tags
if not isinstance(kw.tag, basestring):
continue
if kw.tag in kwTags:
# Mark that we found a required tag
kwTags[kw.tag] = True
# Save the values we'll need into variables for ease of use
if (kw.tag == "kwformat"):
kwformat = kw.text.lower() # lower() for ease of compare
if (kw.tag == "kwlen"):
kwlen = int(kw.text)
if (kw.tag == "kwdata"):
# If it's mixed format, we want kwdata to actually hold all the xml tags contained in this kwdata
# Otherwise, grab the plain text so we can treat it like data later
if (kwformat == "mixed"):
kwdata = kw
else:
kwdata = kw.text
elif kw.tag in kwdTags:
# Ignore the kwdTags for now, we'll check them later
next
else:
# Flag that we found an unsupported tag. This may help catch typos, etc..
out.error("The unsupported tag \"<%s>\" was found in keyword %s in record %s" %
(kw.tag, keywordName, recordName))
errorsFound += 1
# --------
# Make sure all the required kwTags were found
for kw in kwTags:
if (kwTags[kw] == False):
out.error("Required tag \"<%s>\" was not found in keyword %s in record %s" %
(kw, keywordName, recordName))
errorsFound += 1
# Now we know the basics of the template are correct, now do more indepth checking of length, etc..
# --------
# Make sure the keyword is two characters long
if (len(keywordName) != 2):
out.error("The length of the keyword %s in record %s is not 2 characters long" %
(keywordName, recordName))
errorsFound += 1
# --------
# A check to make sure the RT keyword kwdata matches the name of the record we are in
if ((keywordName == "RT") and (recordName != kwdata)):
out.error("The value of the RT keyword \"%s\" does not match the record name \"%s\"" %
(kwdata, recordName))
errorsFound += 1
# --------
# Check that the length specified isn't longer than the keyword supports
# Keywords that start with # are 2 bytes, others are 1 byte
if (keywordName[0] == "#"):
maxlen = 65535
else:
maxlen = 255
if (kwlen > maxlen):
out.error("The specified length %d is bigger than the max length %d for keyword %s in record %s" %
(kwlen, maxlen, keywordName, recordName))
errorsFound += 1
# --------
# If the input format is hex, make sure the input data is hex only
if (kwformat == "hex"):
(rc, kwdata) = checkHexDataFormat(kwdata)
if (rc):
out.error("checkHexDataFormat return an error for for keyword %s in record %s" %
(keywordName, recordName))
errorsFound += 1
# --------
# If the input format is mixed, loop over the kwdata and verify it is formatted properly
if (kwformat == "mixed"):
# We can't use the length check code below for the mixed case, so track it here and check below
kwdatalen = 0
# We need to verify the format and length of the ascii or hex keywords embedded in here
for kwd in kwdata.iter():
# Comments aren't basestring tags
if not isinstance(kwd.tag, basestring):
continue
# Make sure it only contains the two keywords we expect
if kwd.tag.lower() in kwdTags:
if (kwd.tag.lower() == "ascii"):
kwdatalen += len(kwd.text)
if (kwd.tag.lower() == "hex"):
(rc, kwdata) = checkHexDataFormat(kwd.text)
if (rc):
out.error("checkHexDataFormat return an error for for keyword %s in record %s" %
(keywordName, recordName))
errorsFound += 1
# Nibbles to bytes
kwdatalen += (len(kwdata)/2)
elif (kwd.tag.lower() == "kwdata"):
next # Ignore this tag at this level
else:
# Flag that we found an unsupported tag. This may help catch typos, etc..
out.error("The unsupported tag \"<%s>\" was found in kwdata for keyword %s in record %s" %
(kwd.tag, keywordName, recordName))
errorsFound += 1
# Done looping through the tags we found, now check that the length isn't too long
if (kwdatalen > kwlen):
out.error("The total length of the mixed data is longer than the given <kwlen> for keyword %s in record %s" %
(keywordName, recordName))
errorsFound += 1
# --------
# Verify that the data isn't longer than the length given
# Future checks could include making sure bin data is hex
if (kwformat == "ascii"):
if (len(kwdata) > kwlen):
out.error("The length of the value is longer than the given <kwlen> for keyword %s in record %s" %
(keywordName, recordName))
errorsFound += 1
elif (kwformat == "hex"):
# Convert hex nibbles to bytes for len compare
if ((len(kwdata)/2) > kwlen):
out.error("The length of the value is longer than the given <kwlen> for keyword %s in record %s" %
(keywordName, recordName))
errorsFound += 1
elif (kwformat == "mixed"):
# The mixed tag length checking was handled above
next
else:
out.error("Unknown keyword format \"%s\" given for keyword %s in record %s" %
(kwformat, keywordName, recordName))
errorsFound += 1
# Done with the record, reset the output
out.setIndent(2)
# All done with error checks, bailout if we hit something
if (errorsFound):
out.msg("")
out.error("%d error%s found in the tvpd data. Please review the above errors and correct them." %
(errorsFound, "s" if (errorsFound > 1) else ""))
tvpdFileName = os.path.join(clOutputPath, vpdName + "-err.tvpd")
rc = writeXml(manifest, tvpdFileName)
if (rc):
exit(rc)
out.msg("Wrote tvpd file to help in debug: %s" % tvpdFileName)
exit(errorsFound)
# We now have a correct tvpd, use it to create a binary VPD image
out.setIndent(0)
out.msg("==== Stage 3: Creating VPD output files")
out.setIndent(2)
# Create our output file names
if (clRecordMode):
tvpdFileName = os.path.join(clOutputPath, vpdName)
else:
tvpdFileName = os.path.join(clOutputPath, vpdName + ".tvpd")
vpdFileName = os.path.join(clOutputPath, vpdName + ".vpd")
# This is our easy one, write the XML back out
# Write out the full template vpd representing the data contained in our image
rc = writeXml(manifest, tvpdFileName)
if (rc):
exit(rc)
out.msg("Wrote tvpd file: %s" % tvpdFileName)
# In record only mode we don't want to write the binary file, so we bail from the program here
if (clRecordMode):
exit(errorsFound)
# Now the hard part, write out the binary file
# Open up our file to write
vpdFile = open(vpdFileName, "wb")
# Process for creating the binary file
# There are 2 ways the file could be created
# 1 - write the data to file on disk as the records are created
# 2 - write the data to memory and then write the complete image at the end
# Option 2 was selected for a few reasons
# - This isn't a large amount of data where flushing to disk as we went along would help performance
# - When the TOC entries are created, we don't know the offset/length to provide. This has to be updated later
# By keeping the records in memory, it's marginally easier to update the TOC info since you don't have to manage file position
# - While ECC isn't supported now, if it is needed in the future, the entire record will be available in memory for the algoritm
# If writing to the file was done, the data would have to be read back and sent to the ECC algorithm
#
# The rest of this process is pretty straight forward. There are some helper functions implemented at the top of the file
# The most difficult piece is tracking the total image size and managing the TOC offset locations so they can be later updated
# Our dictionary of all the records we've created in memory, along with info like TOC offsets
recordInfo = dict()
# Track total image size as the various records are created
imageSize = 0
# The VHDR and VTOC are created by the tool based upon the info in the tvpd
# The rest of the records are derived from the tvpd
################################################
# Create VHDR
recordName = "VHDR"
recordInfo[recordName] = RecordInfo()
# Create the ECC block
recordInfo[recordName].record += bytearray(bytearray.fromhex("0000000000000000000000"))
# Create the Large Resource Tag
recordInfo[recordName].record += bytearray(bytearray.fromhex("84"))
# Create the Record Length