Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a flag to disable transactional support #133

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Add a flag to disable transactional support #133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8eae467
add a flag to disable transactional support
givanovexpe Oct 16, 2024
2c835d7
fix
givanovexpe Oct 16, 2024
83dcc4c
fix
givanovexpe Oct 16, 2024
43fd06a
test
givanovexpe Oct 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).

## [5.2.2] - 2024-10-16
### Added
- Added `DISABLE_TRANSACTIONS_SUPPORT` option to disable transactional support in Hive metastore

## [5.2.1] - 2024-08-23
### Added
- Upgrade yum repos from EMR-5.36.2 (latest EMR 5 version)
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
| DATANUCLEUS_CONNECTION_POOL_NAME | No | Connection pool name (HikariCP only). |
| DATANUCLEUS_CONNECTION_POOL_CATALOG | No | Connection pool catalog (HikariCP only). |
| DATANUCLEUS_CONNECTION_POOL_REGISTER_MBEANS | No | Register MBeans for the connection pool (HikariCP only). |
| DISABLE_TRANSACTIONS_SUPPORT | No | Option to turn disable transactional support in the Hive Metastore. |
| DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES | No | `true`/`false` value for hive.metastore.disallow.incompatible.col.type.changes, default `true`. |
| ENABLE_GLUESYNC | No | Option to turn on GlueSync Hive Metastore listener. |
| ENABLE_HIVE_LOCK_HOUSE_KEEPER | No | Option to turn on Hive Metastore Hive Lock House Keeper. |
Expand Down Expand Up @@ -68,7 +69,6 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
| RANGER_POLICY_MANAGER_URL | No | Ranger admin URL from where policies will be downloaded. |
| RANGER_SERVICE_NAME | No | Ranger service name used to configure RangerAuth plugin. |
| SNS_ARN | No | The SNS topic ARN to which metadata updates will be
|

# Contact

Expand Down
8 changes: 8 additions & 0 deletions files/startup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@ if [ ! -z ${DATANUCLEUS_CONNECTION_POOLING_TYPE} ]; then
fi
fi

if [[ -n ${DISABLE_TRANSACTIONS_SUPPORT} ]]; then
update_property.py hive.txn.manager org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager /etc/hive/conf/hive-site.xml
update_property.py hive.support.concurrency false /etc/hive/conf/hive-site.xml
update_property.py hive.compactor.initiator.on false /etc/hive/conf/hive-site.xml
update_property.py hive.compactor.worker.threads 0 /etc/hive/conf/hive-site.xml
update_property.py --append hive.conf.restricted.list hive.txn.manager,hive.support.concurrency,hive.compactor.initiator.on,hive.compactor.worker.threads /etc/hive/conf/hive-site.xml
fi

if [[ -n ${DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES} ]]; then
update_property.py hive.metastore.disallow.incompatible.col.type.changes "${DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES}" /etc/hive/conf/hive-site.xml
fi
Expand Down
95 changes: 61 additions & 34 deletions files/update_property.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,42 +25,69 @@
from xml.etree import ElementTree as ET
from xml.parsers.expat import ExpatError

def write_properties_to_xml(xml_path, property_name='', property_value=''):
if(os.path.isfile(xml_path)):
try:
xml = ET.parse(xml_path)
except ExpatError:
print "Error while parsing file:" + xml_path
return -1
property_exists = False
root = xml.getroot()
for child in root.findall('property'):
name = child.find("name").text.strip()
if name == property_name:
property_exists = True
child.find("value").text = property_value
break
if not property_exists:
new_property = ET.SubElement(root, 'property')
ET.SubElement(new_property,'name').text = property_name
ET.SubElement(new_property,'value').text = property_value
def write_properties_to_xml(xml_path, property_name='', property_value='', append=False):
if os.path.isfile(xml_path):
try:
xml = ET.parse(xml_path)
except ExpatError:
print("Error while parsing file:" + xml_path)
return -1

property_exists = False
root = xml.getroot()

for child in root.findall('property'):
name = child.find("name").text.strip()
if name == property_name:
property_exists = True
value_element = child.find("value")

xml.write(xml_path)
return 0
else:
return -1
if value_element is None:
# If <value> does not exist, create it
value_element = ET.SubElement(child, "value")
current_value = ''
else:
current_value = value_element.text if value_element.text else ''
current_value = current_value.strip()

if append:
# Append the new value to the existing one, separated by commas
if current_value:
new_value = current_value + ',' + property_value
else:
new_value = property_value
else:
new_value = property_value

value_element.text = new_value
break

if not property_exists:
new_property = ET.SubElement(root, 'property')
ET.SubElement(new_property, 'name').text = property_name
ET.SubElement(new_property, 'value').text = property_value

xml.write(xml_path)
return 0
else:
return -1



if __name__ == '__main__':
if(len(sys.argv) > 1):
if(len(sys.argv) > 3):
parameter_name = sys.argv[1] if len(sys.argv) > 1 else None
parameter_value = sys.argv[2] if len(sys.argv) > 2 else None
ranger_admin_site_xml_path = sys.argv[3] if len(sys.argv) > 3 else None
else:
if(len(sys.argv) > 2):
parameter_name = sys.argv[1] if len(sys.argv) > 1 else None
parameter_value = ""
ranger_admin_site_xml_path = sys.argv[2] if len(sys.argv) > 2 else None
write_properties_to_xml(ranger_admin_site_xml_path,parameter_name,parameter_value)
if len(sys.argv) > 1:
append_mode = '--append' in sys.argv # Check if '--append' is passed
if append_mode:
sys.argv.remove('--append')

if len(sys.argv) > 3:
parameter_name = sys.argv[1] if len(sys.argv) > 1 else None
parameter_value = sys.argv[2] if len(sys.argv) > 2 else None
file_xml_path = sys.argv[3] if len(sys.argv) > 3 else None
else:
if len(sys.argv) > 2:
parameter_name = sys.argv[1] if len(sys.argv) > 1 else None
parameter_value = ""
file_xml_path = sys.argv[2] if len(sys.argv) > 2 else None

write_properties_to_xml(file_xml_path, parameter_name, parameter_value, append=append_mode)
Loading