Skip to content

adding lamda expression to filter tuple #831

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MariusWirtz merged 3 commits into from
Nov 17, 2022
Merged

adding lamda expression to filter tuple #831

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
308dc21
adding lamda expression to filter tuple
ldelberg Nov 16, 2022
17311bf
added elements dict as parameter
ldelberg Nov 16, 2022
b7d482d
added test for filter by elements
ldelberg Nov 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 57 additions & 28 deletions TM1py/Services/ServerService.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def wrapper(self, *args, **kwargs):

class ServerService(ObjectService):
""" Service to query common information from the TM1 Server

"""

def __init__(self, rest: RestService):
Expand All @@ -54,12 +54,15 @@ def initialize_transaction_log_delta_requests(self, filter=None, **kwargs):
url += "?$filter={}".format(filter)
response = self._rest.GET(url=url, **kwargs)
# Read the next delta-request-url from the response
self.tlog_last_delta_request = response.text[response.text.rfind("TransactionLogEntries/!delta('"):-2]
self.tlog_last_delta_request = response.text[response.text.rfind(
"TransactionLogEntries/!delta('"):-2]

@odata_track_changes_header
def execute_transaction_log_delta_request(self, **kwargs) -> Dict:
response = self._rest.GET(url="/api/v1/" + self.tlog_last_delta_request, **kwargs)
self.tlog_last_delta_request = response.text[response.text.rfind("TransactionLogEntries/!delta('"):-2]
response = self._rest.GET(
url="/api/v1/" + self.tlog_last_delta_request, **kwargs)
self.tlog_last_delta_request = response.text[response.text.rfind(
"TransactionLogEntries/!delta('"):-2]
return response.json()['value']

@odata_track_changes_header
Expand All @@ -69,12 +72,15 @@ def initialize_audit_log_delta_requests(self, filter=None, **kwargs):
url += "?$filter={}".format(filter)
response = self._rest.GET(url=url, **kwargs)
# Read the next delta-request-url from the response
self.alog_last_delta_request = response.text[response.text.rfind("AuditLogEntries/!delta('"):-2]
self.alog_last_delta_request = response.text[response.text.rfind(
"AuditLogEntries/!delta('"):-2]

@odata_track_changes_header
def execute_audit_log_delta_request(self, **kwargs) -> Dict:
response = self._rest.GET(url="/api/v1/" + self.alog_last_delta_request, **kwargs)
self.alog_last_delta_request = response.text[response.text.rfind("AuditLogEntries/!delta('"):-2]
response = self._rest.GET(
url="/api/v1/" + self.alog_last_delta_request, **kwargs)
self.alog_last_delta_request = response.text[response.text.rfind(
"AuditLogEntries/!delta('"):-2]
return response.json()['value']

@odata_track_changes_header
Expand All @@ -84,12 +90,15 @@ def initialize_message_log_delta_requests(self, filter=None, **kwargs):
url += "?$filter={}".format(filter)
response = self._rest.GET(url=url, **kwargs)
# Read the next delta-request-url from the response
self.mlog_last_delta_request = response.text[response.text.rfind("MessageLogEntries/!delta('"):-2]
self.mlog_last_delta_request = response.text[response.text.rfind(
"MessageLogEntries/!delta('"):-2]

@odata_track_changes_header
def execute_message_log_delta_request(self, **kwargs) -> Dict:
response = self._rest.GET(url="/api/v1/" + self.mlog_last_delta_request, **kwargs)
self.mlog_last_delta_request = response.text[response.text.rfind("MessageLogEntries/!delta('"):-2]
response = self._rest.GET(
url="/api/v1/" + self.mlog_last_delta_request, **kwargs)
self.mlog_last_delta_request = response.text[response.text.rfind(
"MessageLogEntries/!delta('"):-2]
return response.json()['value']

@require_admin
Expand All @@ -112,7 +121,8 @@ def get_message_log_entries(self, reverse: bool = True, since: datetime = None,
"""
msg_contains_operator = msg_contains_operator.strip().lower()
if msg_contains_operator not in ("and", "or"):
raise ValueError("'msg_contains_operator' must be either 'AND' or 'OR'")
raise ValueError(
"'msg_contains_operator' must be either 'AND' or 'OR'")

reverse = 'desc' if reverse else 'asc'
url = '/api/v1/MessageLogEntries?$orderby=TimeStamp {}'.format(reverse)
Expand All @@ -124,13 +134,15 @@ def get_message_log_entries(self, reverse: bool = True, since: datetime = None,
# If since doesn't have tz information, UTC is assumed
if not since.tzinfo:
since = self.utc_localize_time(since)
log_filters.append(format_url("TimeStamp ge {}", since.strftime("%Y-%m-%dT%H:%M:%SZ")))
log_filters.append(format_url(
"TimeStamp ge {}", since.strftime("%Y-%m-%dT%H:%M:%SZ")))

if until:
# If until doesn't have tz information, UTC is assumed
if not until.tzinfo:
until = self.utc_localize_time(until)
log_filters.append(format_url("TimeStamp le {}", until.strftime("%Y-%m-%dT%H:%M:%SZ")))
log_filters.append(format_url(
"TimeStamp le {}", until.strftime("%Y-%m-%dT%H:%M:%SZ")))

if logger:
log_filters.append(format_url("Logger eq '{}'", logger))
Expand All @@ -144,11 +156,13 @@ def get_message_log_entries(self, reverse: bool = True, since: datetime = None,

if msg_contains:
if isinstance(msg_contains, str):
log_filters.append(format_url("contains(toupper(Message),toupper('{}'))", msg_contains))
log_filters.append(format_url(
"contains(toupper(Message),toupper('{}'))", msg_contains))
else:
msg_filters = [format_url("contains(toupper(Message),toupper('{}'))", wildcard)
for wildcard in msg_contains]
log_filters.append("({})".format(f" {msg_contains_operator} ".join(msg_filters)))
log_filters.append("({})".format(
f" {msg_contains_operator} ".join(msg_filters)))

url += "&$filter={}".format(" and ".join(log_filters))

Expand All @@ -164,19 +178,23 @@ def write_to_message_log(self, level: str, message: str, **kwargs) -> None:
:param level: string, FATAL, ERROR, WARN, INFO, DEBUG
:param message: string
:return:
"""
"""

valid_levels = CaseAndSpaceInsensitiveSet({'FATAL', 'ERROR', 'WARN', 'INFO', 'DEBUG'})
valid_levels = CaseAndSpaceInsensitiveSet(
{'FATAL', 'ERROR', 'WARN', 'INFO', 'DEBUG'})
if level not in valid_levels:
raise ValueError(f"Invalid level: '{level}'")

from TM1py.Services import ProcessService
process_service = ProcessService(self._rest)
process = Process(name="", prolog_procedure="LogOutput('{}', '{}');".format(level, message))
success, status, _ = process_service.execute_process_with_return(process, **kwargs)
process = Process(
name="", prolog_procedure="LogOutput('{}', '{}');".format(level, message))
success, status, _ = process_service.execute_process_with_return(
process, **kwargs)

if not success:
raise RuntimeError(f"Failed to write to TM1 Message Log through unbound process. Status: '{status}'")
raise RuntimeError(
f"Failed to write to TM1 Message Log through unbound process. Status: '{status}'")

@staticmethod
def utc_localize_time(timestamp):
Expand All @@ -185,36 +203,43 @@ def utc_localize_time(timestamp):
return timestamp_utc

@require_admin
def get_transaction_log_entries(self, reverse: bool = True, user: str = None, cube: str = None,
def get_transaction_log_entries(self, reverse: bool = True, user: str = None, cube: str = None, elements: Dict = None,
since: datetime = None, until: datetime = None, top: int = None, **kwargs) -> Dict:
"""
:param reverse: Boolean
:param user: UserName
:param cube: CubeName
:param elements: of type dict. Filtervalue as key and comparison operator as value tuple={'Filtervalue1':'eq','Filtervalue2': 'ge'}
:param since: of type datetime. If it doesn't have tz information, UTC is assumed.
:param until: of type datetime. If it doesn't have tz information, UTC is assumed.
:param top: int
:return:
"""
reverse = 'desc' if reverse else 'asc'
url = '/api/v1/TransactionLogEntries?$orderby=TimeStamp {} '.format(reverse)
url = '/api/v1/TransactionLogEntries?$orderby=TimeStamp {} '.format(
reverse)
# filter on user, cube and time
if user or cube or since or until:
log_filters = []
if user:
log_filters.append(format_url("User eq '{}'", user))
if cube:
log_filters.append(format_url("Cube eq '{}'", cube))
if elements:
log_filters.append(format_url(
"Tuple/any(t: {})".format(" or ".join([f"t {v} '{k}'" for k, v in elements.items()]))))
if since:
# If since doesn't have tz information, UTC is assumed
if not since.tzinfo:
since = self.utc_localize_time(since)
log_filters.append(format_url("TimeStamp ge {}", since.strftime("%Y-%m-%dT%H:%M:%SZ")))
log_filters.append(format_url(
"TimeStamp ge {}", since.strftime("%Y-%m-%dT%H:%M:%SZ")))
if until:
# If until doesn't have tz information, UTC is assumed
if not until.tzinfo:
until = self.utc_localize_time(until)
log_filters.append(format_url("TimeStamp le {}", until.strftime("%Y-%m-%dT%H:%M:%SZ")))
log_filters.append(format_url(
"TimeStamp le {}", until.strftime("%Y-%m-%dT%H:%M:%SZ")))
url += "&$filter={}".format(" and ".join(log_filters))
# top limit
if top:
Expand Down Expand Up @@ -243,19 +268,23 @@ def get_audit_log_entries(self, user: str = None, object_type: str = None, objec
if user:
log_filters.append(format_url("UserName eq '{}'", user))
if object_type:
log_filters.append(format_url("ObjectType eq '{}'", object_type))
log_filters.append(format_url(
"ObjectType eq '{}'", object_type))
if object_name:
log_filters.append(format_url("ObjectName eq '{}'", object_name))
log_filters.append(format_url(
"ObjectName eq '{}'", object_name))
if since:
# If since doesn't have tz information, UTC is assumed
if not since.tzinfo:
since = self.utc_localize_time(since)
log_filters.append(format_url("TimeStamp ge {}", since.strftime("%Y-%m-%dT%H:%M:%SZ")))
log_filters.append(format_url(
"TimeStamp ge {}", since.strftime("%Y-%m-%dT%H:%M:%SZ")))
if until:
# If until doesn't have tz information, UTC is assumed
if not until.tzinfo:
until = self.utc_localize_time(until)
log_filters.append(format_url("TimeStamp le {}", until.strftime("%Y-%m-%dT%H:%M:%SZ")))
log_filters.append(format_url(
"TimeStamp le {}", until.strftime("%Y-%m-%dT%H:%M:%SZ")))
url += "&$filter={}".format(" and ".join(log_filters))
# top limit
if top:
Expand Down
74 changes: 54 additions & 20 deletions Tests/ServerService_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,18 @@ def setUpClass(cls):
cls.tm1.dimensions.update_or_create(d)

if not cls.tm1.cubes.exists(cls.cube_name):
cube = Cube(cls.cube_name, [cls.dimension_name1, cls.dimension_name2])
cube = Cube(cls.cube_name, [
cls.dimension_name1, cls.dimension_name2])
cls.tm1.cubes.update_or_create(cube)

# inject process with ItemReject
cls.process1 = Process(name=cls.process_name1, prolog_procedure="ItemReject('TM1py Tests');")
cls.process1 = Process(name=cls.process_name1,
prolog_procedure="ItemReject('TM1py Tests');")
cls.tm1.processes.update_or_create(cls.process1)

# inject process that does nothing and runs successful
cls.process2 = Process(name=cls.process_name2, prolog_procedure="sText = 'text';")
cls.process2 = Process(name=cls.process_name2,
prolog_procedure="sText = 'text';")
cls.tm1.processes.update_or_create(cls.process2)

cls.tm1.server.activate_audit_log()
Expand Down Expand Up @@ -92,7 +95,8 @@ def test_get_data_directory(self):
self.assertGreater(len(data_directory), 0)

active_configuration = self.tm1.server.get_active_configuration()
self.assertEqual(data_directory, active_configuration["Administration"]["DataBaseDirectory"])
self.assertEqual(
data_directory, active_configuration["Administration"]["DataBaseDirectory"])

def test_get_static_configuration(self):
static_configuration = self.tm1.server.get_static_configuration()
Expand All @@ -118,7 +122,8 @@ def test_update_static_configuration(self):
}
}
}
response = self.tm1.server.update_static_configuration(config_changes)
response = self.tm1.server.update_static_configuration(
config_changes)
self.assertTrue(response.ok)

active_config = self.tm1.server.get_active_configuration()
Expand All @@ -137,19 +142,22 @@ def test_get_last_process_message_from_message_log(self):
raise e
# TM1 takes one second to write to the message-log
time.sleep(1)
log_entry = self.tm1.server.get_last_process_message_from_messagelog(self.process_name1)
log_entry = self.tm1.server.get_last_process_message_from_messagelog(
self.process_name1)
regex = re.compile('TM1ProcessError_.*.log')
self.assertTrue(regex.search(log_entry))

self.tm1.processes.execute(self.process_name2)
# TM1 takes one second to write to the message-log
time.sleep(1)
log_entry = self.tm1.server.get_last_process_message_from_messagelog(self.process_name2)
log_entry = self.tm1.server.get_last_process_message_from_messagelog(
self.process_name2)
regex = re.compile('TM1ProcessError_.*.log')
self.assertFalse(regex.search(log_entry))

def test_get_last_transaction_log_entries(self):
self.tm1.processes.execute_ti_code(lines_prolog="CubeSetLogChanges('{}', {});".format(self.cube_name, 1))
self.tm1.processes.execute_ti_code(
lines_prolog="CubeSetLogChanges('{}', {});".format(self.cube_name, 1))

tmstp = datetime.datetime.utcnow()

Expand Down Expand Up @@ -208,10 +216,26 @@ def test_get_last_transaction_log_entries(self):
for v1, v2, v3 in zip(random_values, reversed(values_from_top), reversed(values_from_since)):
self.assertAlmostEqual(v1, v2, delta=0.000000001)

# Query transaction log with Since and Elements filter
entries = self.tm1.server.get_transaction_log_entries(
reverse=True,
cube=cube,
elements={'2001': 'eq'},
since=tmstp,
top=10)
values_from_elements = [entry['NewValue'] for entry in entries]
self.assertEqual(len(values_from_elements), 1)

# Compare value written to cube vs. value from filtered log
# second value written to cube was ('2001', 'Value'): random_values[1]
self.assertAlmostEqual(values_from_elements[0], random_values[1])

def test_get_transaction_log_entries_from_today(self):
# get datetime from today at 00:00:00
today = datetime.datetime.combine(datetime.date.today(), datetime.time(0, 0))
entries = self.tm1.server.get_transaction_log_entries(reverse=True, since=today)
today = datetime.datetime.combine(
datetime.date.today(), datetime.time(0, 0))
entries = self.tm1.server.get_transaction_log_entries(
reverse=True, since=today)
self.assertTrue(len(entries) > 0)
for entry in entries:
entry_timestamp = parser.parse(entry['TimeStamp'])
Expand All @@ -222,7 +246,8 @@ def test_get_transaction_log_entries_from_today(self):

def test_get_audit_log_entries_from_today(self):
# get datetime from today at 00:00:00
today = datetime.datetime.combine(datetime.date.today(), datetime.time(0, 0))
today = datetime.datetime.combine(
datetime.date.today(), datetime.time(0, 0))
entries = self.tm1.server.get_audit_log_entries(since=today)
self.assertTrue(len(entries) > 0)
for entry in entries:
Expand Down Expand Up @@ -253,8 +278,10 @@ def test_get_audit_log_entries_top(self):

def test_get_transaction_log_entries_until_yesterday(self):
# get datetime until yesterday at 00:00:00
yesterday = datetime.datetime.combine(datetime.date.today() - timedelta(days=1), datetime.time(0, 0))
entries = self.tm1.server.get_transaction_log_entries(reverse=True, until=yesterday)
yesterday = datetime.datetime.combine(
datetime.date.today() - timedelta(days=1), datetime.time(0, 0))
entries = self.tm1.server.get_transaction_log_entries(
reverse=True, until=yesterday)
self.assertTrue(len(entries) > 0)
for entry in entries:
# skip invalid timestamps from log
Expand All @@ -268,8 +295,10 @@ def test_get_transaction_log_entries_until_yesterday(self):

def test_get_message_log_entries_from_today(self):
# get datetime from today at 00:00:00
today = datetime.datetime.combine(datetime.date.today(), datetime.time(0, 0))
entries = self.tm1.server.get_message_log_entries(reverse=True, since=today)
today = datetime.datetime.combine(
datetime.date.today(), datetime.time(0, 0))
entries = self.tm1.server.get_message_log_entries(
reverse=True, since=today)

for entry in entries:
entry_timestamp = parser.parse(entry['TimeStamp'])
Expand All @@ -280,9 +309,11 @@ def test_get_message_log_entries_from_today(self):

def test_get_message_log_entries_until_yesterday(self):
# get datetime until yesterday at 00:00:00
yesterday = datetime.datetime.combine(datetime.date.today() - timedelta(days=1), datetime.time(0, 0))
yesterday = datetime.datetime.combine(
datetime.date.today() - timedelta(days=1), datetime.time(0, 0))

entries = self.tm1.server.get_message_log_entries(reverse=True, until=yesterday)
entries = self.tm1.server.get_message_log_entries(
reverse=True, until=yesterday)
self.assertTrue(len(entries) > 0)
for entry in entries:
# skip invalid timestamps from log
Expand All @@ -296,10 +327,13 @@ def test_get_message_log_entries_until_yesterday(self):

def test_get_message_log_entries_only_yesterday(self):
# get datetime only yesterday at 00:00:00
yesterday = datetime.datetime.combine(datetime.date.today() - timedelta(days=1), datetime.time(0, 0))
today = datetime.datetime.combine(datetime.date.today() - timedelta(days=1), datetime.time(0, 0))
yesterday = datetime.datetime.combine(
datetime.date.today() - timedelta(days=1), datetime.time(0, 0))
today = datetime.datetime.combine(
datetime.date.today() - timedelta(days=1), datetime.time(0, 0))

entries = self.tm1.server.get_message_log_entries(reverse=True, since=yesterday, until=today)
entries = self.tm1.server.get_message_log_entries(
reverse=True, since=yesterday, until=today)
for entry in entries:
entry_timestamp = parser.parse(entry['TimeStamp'])
entry_date = entry_timestamp.date()
Expand Down