Skip to content

Commit

Permalink
add a flag to disable transactional support
Browse files Browse the repository at this point in the history
  • Loading branch information
@givanovexpe
givanovexpe committed Oct 16, 2024
1 parent ab1cdc5 commit 8eae467
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 36 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).

## [5.2.2] - 2024-10-16
### Added
- Added `DISABLE_TRANSACTIONS_SUPPORT` option to disable transactional support in Hive metastore

## [5.2.1] - 2024-08-23
### Added
- Upgrade yum repos from EMR-5.36.2 (latest EMR 5 version)
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
| DATANUCLEUS_CONNECTION_POOL_NAME | No | Connection pool name (HikariCP only). |
| DATANUCLEUS_CONNECTION_POOL_CATALOG | No | Connection pool catalog (HikariCP only). |
| DATANUCLEUS_CONNECTION_POOL_REGISTER_MBEANS | No | Register MBeans for the connection pool (HikariCP only). |
| DISABLE_TRANSACTIONS_SUPPORT | No | Option to turn disable transactional support in the Hive Metastore. |
| DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES | No | `true`/`false` value for hive.metastore.disallow.incompatible.col.type.changes, default `true`. |
| ENABLE_GLUESYNC | No | Option to turn on GlueSync Hive Metastore listener. |
| ENABLE_HIVE_LOCK_HOUSE_KEEPER | No | Option to turn on Hive Metastore Hive Lock House Keeper. |
Expand Down Expand Up @@ -68,7 +69,6 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
| RANGER_POLICY_MANAGER_URL | No | Ranger admin URL from where policies will be downloaded. |
| RANGER_SERVICE_NAME | No | Ranger service name used to configure RangerAuth plugin. |
| SNS_ARN | No | The SNS topic ARN to which metadata updates will be
|

# Contact

Expand Down
8 changes: 8 additions & 0 deletions files/startup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@ if [ ! -z ${DATANUCLEUS_CONNECTION_POOLING_TYPE} ]; then
fi
fi

if [[ -n ${DISABLE_TRANSACTIONS_SUPPORT} ]]; then
update_property.py hive.txn.manager org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager /etc/hive/conf/hive-site.xml
update_property.py hive.support.concurrency false /etc/hive/conf/hive-site.xml
update_property.py hive.compactor.initiator.on false /etc/hive/conf/hive-site.xml
update_property.py hive.compactor.worker.threads 0 /etc/hive/conf/hive-site.xml
update_property.py --append hive.conf.restricted.list hive.txn.manager,hive.support.concurrency,hive.compactor.initiator.on,hive.compactor.worker.threads /etc/hive/conf/hive-site.xml
fi

if [[ -n ${DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES} ]]; then
update_property.py hive.metastore.disallow.incompatible.col.type.changes "${DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES}" /etc/hive/conf/hive-site.xml
fi
Expand Down
89 changes: 54 additions & 35 deletions files/update_property.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,42 +25,61 @@
from xml.etree import ElementTree as ET
from xml.parsers.expat import ExpatError

def write_properties_to_xml(xml_path, property_name='', property_value=''):
if(os.path.isfile(xml_path)):
try:
xml = ET.parse(xml_path)
except ExpatError:
print "Error while parsing file:" + xml_path
return -1
property_exists = False
root = xml.getroot()
for child in root.findall('property'):
name = child.find("name").text.strip()
if name == property_name:
property_exists = True
child.find("value").text = property_value
break
if not property_exists:
new_property = ET.SubElement(root, 'property')
ET.SubElement(new_property,'name').text = property_name
ET.SubElement(new_property,'value').text = property_value

xml.write(xml_path)
return 0
else:
return -1
def write_properties_to_xml(xml_path, property_name='', property_value='', append=False):
if os.path.isfile(xml_path):
try:
xml = ET.parse(xml_path)
except ExpatError:
print("Error while parsing file:" + xml_path)
return -1

property_exists = False
root = xml.getroot()

for child in root.findall('property'):
name = child.find("name").text.strip()
if name == property_name:
property_exists = True
current_value = child.find("value").text.strip()

if append:
# Append the new value to the existing one, separated by commas
if current_value:
new_value = current_value + ',' + property_value
else:
new_value = property_value
else:
new_value = property_value

child.find("value").text = new_value
break

if not property_exists:
new_property = ET.SubElement(root, 'property')
ET.SubElement(new_property, 'name').text = property_name
ET.SubElement(new_property, 'value').text = property_value

xml.write(xml_path)
return 0
else:
print(f"File not found: {xml_path}")
return -1


if __name__ == '__main__':
if(len(sys.argv) > 1):
if(len(sys.argv) > 3):
parameter_name = sys.argv[1] if len(sys.argv) > 1 else None
parameter_value = sys.argv[2] if len(sys.argv) > 2 else None
ranger_admin_site_xml_path = sys.argv[3] if len(sys.argv) > 3 else None
else:
if(len(sys.argv) > 2):
parameter_name = sys.argv[1] if len(sys.argv) > 1 else None
parameter_value = ""
ranger_admin_site_xml_path = sys.argv[2] if len(sys.argv) > 2 else None
write_properties_to_xml(ranger_admin_site_xml_path,parameter_name,parameter_value)
if len(sys.argv) > 1:
append_mode = '--append' in sys.argv # Check if '--append' is passed
if append_mode:
sys.argv.remove('--append')

if len(sys.argv) > 3:
parameter_name = sys.argv[1] if len(sys.argv) > 1 else None
parameter_value = sys.argv[2] if len(sys.argv) > 2 else None
ranger_admin_site_xml_path = sys.argv[3] if len(sys.argv) > 3 else None
else:
if len(sys.argv) > 2:
parameter_name = sys.argv[1] if len(sys.argv) > 1 else None
parameter_value = ""
file_xml_path = sys.argv[2] if len(sys.argv) > 2 else None

write_properties_to_xml(file_xml_path, parameter_name, parameter_value, append=append_mode)

0 comments on commit 8eae467

Please sign in to comment.