Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/candidate-9.8.x'
Browse files Browse the repository at this point in the history 
Signed-off-by: Gordon Smith <[email protected]>

# Conflicts:
#	helm/hpcc/Chart.yaml
#	helm/hpcc/templates/_helpers.tpl
#	version.cmake
  • Loading branch information
@GordonSmith
GordonSmith committed Aug 23, 2024
2 parents acae664 + 3e82e1f commit 158b5a8
Show file tree
Hide file tree
Showing 74 changed files with 1,026 additions and 408 deletions.
80 changes: 41 additions & 39 deletions .github/workflows/jirabot.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,33 +38,61 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GHUB_JIRA_USER_MAP: ${{ vars.GHUB_JIRA_USER_MAP }}
JIRA_ISSUE_PROPERTY_MAP: ${{ vars.JIRA_ISSUE_PROPERTY_MAP }}
JIRA_ISSUE_TRANSITION_MAP: ${{ vars.JIRA_ISSUE_TRANSITION_MAP }}
run: |
import os
import re
import time
import sys
import json
import subprocess
from email.utils import parseaddr
from atlassian.jira import Jira
def updateIssue(jira, issue, prAuthor : str, transitionMap: dict, propertyMap: dict, pull_url: str) -> str:
def sanitizeInput(input: str, inputType: str) -> str:
if inputType.lower() == 'email':
# Return the email address only, returns '' if not valid or found
return parseaddr(input)[1]
else:
return ''
def updateIssue(jira, issue, prAuthor : str, propertyMap: dict, pull_url: str) -> str:
result = ''
issueName = issue['key']
issueFields = issue['fields']
statusName = str(issueFields['status']['name'])
transition = transitionMap.get(statusName, None)
# Need to update user first in case we are starting from Unresourced
if prAuthor:
assignee = issueFields['assignee']
if assignee is None:
assigneeId = ''
assigneeEmail = ''
else:
assigneeId = assignee['accountId']
assigneeEmail = assignee["emailAddress"]
assigneeEmail = sanitizeInput(assigneeEmail, 'email')
prAuthorId = prAuthor["accountId"]
prAuthorEmail = prAuthor["emailAddress"]
prAuthorEmail = sanitizeInput(prAuthorEmail, 'email')
if assigneeId is None or assigneeId == '':
jira.assign_issue(issueName, prAuthorId)
result += 'Assigning user: ' + prAuthorEmail + '\n'
elif assigneeId != prAuthorId:
result += 'Changing assignee from: ' + assigneeEmail + ' to: ' + prAuthorEmail + '\n'
jira.assign_issue(issueName, prAuthorId)
if transition == None:
print('Error: Unable to find transition for status: ' + statusName)
elif transition != '':
transitionFlow = ['Merge Pending']
for desiredStatus in transitionFlow:
try:
jira.issue_transition(issueName, transition)
result += 'Workflow Transition: ' + transition + '\n'
transitionId = jira.get_transition_id_to_status_name(issueName, desiredStatus)
jira.set_issue_status_by_transition_id(issueName, transitionId)
result += 'Workflow Transition To: ' + desiredStatus + '\n'
except Exception as error:
transitions = jira.get_issue_transitions(issueName)
result += 'Error: Transition: "' + transition + '" failed with: "' + str(error) + '" Valid transitions=' + str(transitions) + '\n'
result += 'Error: Transitioning to: "' + desiredStatus + '" failed with: "' + str(error) + '" Valid transitions=' + str(transitions) + '\n'
prFieldName = propertyMap.get('pullRequestFieldName', 'customfield_10010')
Expand All @@ -80,24 +108,6 @@ jobs:
elif currentPR is not None and currentPR != pull_url:
result += 'Additional PR: ' + pull_url + '\n'
if prAuthor:
assignee = issueFields['assignee']
if assignee is None:
assigneeId = ''
assigneeEmail = ''
else:
assigneeId = assignee['accountId']
assigneeEmail = assignee["emailAddress"]
prAuthorId = prAuthor["accountId"]
prAuthorEmail = prAuthor["emailAddress"]
if assigneeId is None or assigneeId == '':
jira.assign_issue(issueName, prAuthorId)
result += 'Assigning user: ' + prAuthorEmail + '\n'
elif assigneeId != prAuthorId:
result += 'Changing assignee from: ' + assigneeEmail + ' to: ' + prAuthorEmail + '\n'
jira.assign_issue(issueName, prAuthorId)
return result
jirabot_user = os.environ['JIRABOT_USERNAME']
Expand All @@ -110,7 +120,6 @@ jobs:
github_token = os.environ['GITHUB_TOKEN']
comments_url = os.environ['COMMENTS_URL']
print("%s %s %s" % (title, prAuthor, comments_url))
result = ''
issuem = re.search("(HPCC|HH|IDE|EPE|ML|HPCC4J|JAPI)-[0-9]+", title)
if issuem:
Expand All @@ -131,7 +140,7 @@ jobs:
if userSearchResults and len(userSearchResults) > 0:
jiraUser = userSearchResults[0]
else:
print('Error: Unable to find Jira user: ' + prAuthor + ' continuing without assigning')
print('Error: Unable to map GitHub user to Jira user, continuing without assigning')
if not jira.issue_exists(issue_name):
sys.exit('Error: Unable to find Jira issue: ' + issue_name)
Expand All @@ -140,27 +149,20 @@ jobs:
result = 'Jirabot Action Result:\n'
transitionMap = json.loads(os.environ['JIRA_ISSUE_TRANSITION_MAP'])
if not isinstance(transitionMap, dict):
print('Error: JIRA_ISSUE_TRANSITION_MAP is not a valid JSON object, ignoring.')
transitionMap = {}
jiraIssuePropertyMap = json.loads(os.environ['JIRA_ISSUE_PROPERTY_MAP'])
if not isinstance(jiraIssuePropertyMap, dict):
print('Error: JIRA_ISSUE_PROPERTY_MAP is not a valid JSON object, ignoring.')
jiraIssuePropertyMap = {}
result += updateIssue(jira, issue, jiraUser, transitionMap, jiraIssuePropertyMap, pull_url)
result += updateIssue(jira, issue, jiraUser, jiraIssuePropertyMap, pull_url)
jira.issue_add_comment(issue_name, result)
result = 'Jira Issue: ' + jira_url + '/browse/' + issue_name + '\n\n' + result
# Escape the result for JSON
result = json.dumps(result)
curlCommand = 'curl -X POST %s -H "Content-Type: application/json" -H "Authorization: token %s" --data \'{ "body": %s }\'' % ( comments_url, github_token, result )
print(curlCommand)
os.system(curlCommand)
subprocess.run(['curl', '-X', 'POST', comments_url, '-H', 'Content-Type: application/json', '-H', f'Authorization: token {github_token}', '--data', f'{{ "body": {result} }}'], check=True)
else:
print('Unable to find Jira issue name in title')
Expand Down
2 changes: 2 additions & 0 deletions common/remote/rmtssh.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,8 @@ class CFRunSSH: public CInterface, implements IFRunSSH
printf("%s\n",cmdline.str());
else {
Owned<IPipeProcess> pipe = createPipeProcess();
// reset LD_LIBRARY_PATH here so ssh cmd itself doesn't use HPCC libssl/crypto as they may be different
pipe->setenv("LD_LIBRARY_PATH", ":");
if (pipe->run((verbose&&!usepssh)?"FRUNSSH":NULL,cmdline.str(),workdir,
useplink, // for some reason plink needs input handle
true,true)) {
Expand Down
55 changes: 47 additions & 8 deletions common/workunit/workunit.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4340,6 +4340,8 @@ class CLockedWorkUnit : implements ILocalWorkUnit, implements IExtendedWUInterfa
{ return c->getStateEx(str); }
virtual __int64 getAgentSession() const
{ return c->getAgentSession(); }
virtual __int64 getEngineSession() const
{ return c->getEngineSession(); }
virtual unsigned getAgentPID() const
{ return c->getAgentPID(); }
virtual const char *queryStateDesc() const
Expand Down Expand Up @@ -4498,6 +4500,8 @@ class CLockedWorkUnit : implements ILocalWorkUnit, implements IExtendedWUInterfa
{ c->setStateEx(text); }
virtual void setAgentSession(__int64 sessionId)
{ c->setAgentSession(sessionId); }
virtual void setEngineSession(__int64 sessionId)
{ c->setEngineSession(sessionId); }
virtual void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction)
{ c->setStatistic(creatorType, creator, scopeType, scope, kind, optDescription, value, count, maxValue, mergeAction); }
virtual void setTracingValue(const char * propname, const char * value)
Expand Down Expand Up @@ -5921,9 +5925,11 @@ void CWorkUnitFactory::clearAborting(const char *wuid)
}
}

void CWorkUnitFactory::reportAbnormalTermination(const char *wuid, WUState &state, SessionId agent)
void CWorkUnitFactory::reportAbnormalTermination(const char *wuid, WUState &state, SessionId sessionId, const char *sessionText)
{
WARNLOG("reportAbnormalTermination: session stopped unexpectedly: %" I64F "d state: %d", (__int64) agent, (int) state);
StringBuffer sessionMessage(sessionText);
sessionMessage.appendf(" session stopped unexpectedly [sessionId=%" I64F "d]", (__int64) sessionId);
WARNLOG("reportAbnormalTermination: %s - state: %d", sessionText, (int) state);
bool isEcl = false;
switch (state)
{
Expand All @@ -5940,7 +5946,10 @@ void CWorkUnitFactory::reportAbnormalTermination(const char *wuid, WUState &stat
wu->setState(state);
Owned<IWUException> e = wu->createException();
e->setExceptionCode(isEcl ? 1001 : 1000);
e->setExceptionMessage(isEcl ? "EclCC terminated unexpectedly" : "Workunit terminated unexpectedly");
StringBuffer exceptionText;
exceptionText.append(isEcl ? "EclCC terminated unexpectedly" : "Workunit terminated unexpectedly");
exceptionText.append(" (").append(sessionMessage).append(")");
e->setExceptionMessage(exceptionText);
}

static CriticalSection deleteDllLock;
Expand Down Expand Up @@ -6425,8 +6434,11 @@ class CDaliWorkUnitFactory : public CWorkUnitFactory, implements IDaliClientShut
LocalIAbortHandler abortHandler(*waiter);
if (conn)
{
SessionId agent = -1;
SessionId agentSessionID = -1;
SessionId engineSessionID = -1;
bool agentSessionStopped = false;
bool engineSessionStopped = false;
bool queryRuntimeSessionStopped = false;
unsigned start = msTick();
for (;;)
{
Expand Down Expand Up @@ -6455,22 +6467,37 @@ class CDaliWorkUnitFactory : public CWorkUnitFactory, implements IDaliClientShut
case WUStateAborting:
if (agentSessionStopped)
{
reportAbnormalTermination(wuid, ret, agent);
reportAbnormalTermination(wuid, ret, agentSessionID, "Agent");
return ret;
}
if (engineSessionStopped)
{
reportAbnormalTermination(wuid, ret, engineSessionID, "Engine");
return ret;
}
if (queryDaliServerVersion().compare("2.1")>=0)
{
agent = conn->queryRoot()->getPropInt64("@agentSession", -1);
if((agent>0) && querySessionManager().sessionStopped(agent, 0))
agentSessionID = conn->queryRoot()->getPropInt64("@agentSession", -1);
if((agentSessionID>0) && querySessionManager().sessionStopped(agentSessionID, 0))
{
agentSessionStopped = true;
conn->reload();
continue;
}
engineSessionID = conn->queryRoot()->getPropInt64("@engineSession", -1);
if((engineSessionID>0) && querySessionManager().sessionStopped(engineSessionID, 0))
{
engineSessionStopped = true;
conn->reload();
continue;
}
}
break;
}
agentSessionStopped = false; // reset for state changes such as WUStateWait then WUStateRunning again
// reset for state changes such as WUStateWait then WUStateRunning again
agentSessionStopped = false;
engineSessionStopped = false;

unsigned waited = msTick() - start;
if (timeout==-1 || waited + 20000 < timeout)
{
Expand Down Expand Up @@ -7697,6 +7724,12 @@ void CLocalWorkUnit::setAgentSession(__int64 sessionId)
p->setPropInt64("@agentSession", sessionId);
}

void CLocalWorkUnit::setEngineSession(__int64 sessionId)
{
CriticalBlock block(crit);
p->setPropInt64("@engineSession", sessionId);
}

bool CLocalWorkUnit::getIsQueryService() const
{
CriticalBlock block(crit);
Expand Down Expand Up @@ -7798,6 +7831,12 @@ __int64 CLocalWorkUnit::getAgentSession() const
return p->getPropInt64("@agentSession", -1);
}

__int64 CLocalWorkUnit::getEngineSession() const
{
CriticalBlock block(crit);
return p->getPropInt64("@engineSession", -1);
}

unsigned CLocalWorkUnit::getAgentPID() const
{
CriticalBlock block(crit);
Expand Down
2 changes: 2 additions & 0 deletions common/workunit/workunit.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1286,6 +1286,7 @@ interface IConstWorkUnit : extends IConstWorkUnitInfo
virtual IStringVal & getWorkunitDistributedAccessToken(IStringVal & datoken) const = 0;
virtual IStringVal & getStateEx(IStringVal & str) const = 0;
virtual __int64 getAgentSession() const = 0;
virtual __int64 getEngineSession() const = 0;
virtual unsigned getAgentPID() const = 0;
virtual IConstWUResult * getTemporaryByName(const char * name) const = 0;
virtual IConstWUResultIterator & getTemporaries() const = 0;
Expand Down Expand Up @@ -1376,6 +1377,7 @@ interface IWorkUnit : extends IConstWorkUnit
virtual void setState(WUState state) = 0;
virtual void setStateEx(const char * text) = 0; // Indicates why blocked
virtual void setAgentSession(__int64 sessionId) = 0;
virtual void setEngineSession(__int64 sessionId) = 0;
virtual void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction) = 0;
virtual void setTracingValue(const char * propname, const char * value) = 0;
virtual void setTracingValueInt(const char * propname, int value) = 0;
Expand Down
4 changes: 3 additions & 1 deletion common/workunit/workunit.ipp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,7 @@ public:
virtual WUState getState() const;
virtual IStringVal & getStateEx(IStringVal & str) const;
virtual __int64 getAgentSession() const;
virtual __int64 getEngineSession() const;
virtual unsigned getAgentPID() const;
virtual const char *queryStateDesc() const;
virtual IConstWUResult * getTemporaryByName(const char * name) const;
Expand Down Expand Up @@ -336,6 +337,7 @@ public:
void setState(WUState state);
void setStateEx(const char * text);
void setAgentSession(__int64 sessionId);
void setEngineSession(__int64 sessionId);
bool setDistributedAccessToken(const char * user);
void setStatistic(StatisticCreatorType creatorType, const char * creator, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * optDescription, unsigned __int64 value, unsigned __int64 count, unsigned __int64 maxValue, StatsMergeAction mergeAction);
void setTracingValue(const char * propname, const char * value);
Expand Down Expand Up @@ -624,7 +626,7 @@ public:
}

protected:
void reportAbnormalTermination(const char *wuid, WUState &state, SessionId agent);
void reportAbnormalTermination(const char *wuid, WUState &state, SessionId agent, const char *sessionText);

// These need to be implemented by the derived classes
virtual CLocalWorkUnit* _createWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser) = 0;
Expand Down
2 changes: 2 additions & 0 deletions dali/base/daclient.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,8 @@ void connectLogMsgManagerToDali()

void disconnectLogMsgManagerFromDali()
{
if (isContainerized())
return; // we do not redirect logging between components in containerized environments (this is used for audit->dali in BM)
disconnectLogMsgManagerFromParentOwn(daliClientLoggingParent);
daliClientLoggingParent = 0;
}
Expand Down
Loading

0 comments on commit 158b5a8

Please sign in to comment.