From ea10324292d1c7224c7037c8e872039492a44b2a Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 9 Oct 2024 17:00:56 +0100 Subject: [PATCH 1/7] Add quota command to transactions.py and nlds_client.py --- nlds_client/clientlib/transactions.py | 77 +++++++++++++++++++++++++++ nlds_client/nlds_client.py | 51 +++++++++++++++++- 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/nlds_client/clientlib/transactions.py b/nlds_client/clientlib/transactions.py index a2188bb..8a04387 100644 --- a/nlds_client/clientlib/transactions.py +++ b/nlds_client/clientlib/transactions.py @@ -939,6 +939,83 @@ def change_metadata(user: str, return response_dict +def get_quota(token: str, + user: str, + group: str, + label: str=None, + holding_id: int=None, + transaction_id: str=None, + tag: dict=None): + """Make a request to get the quota for a particular group in the NLDS + :param token: the auth token needed to get the quota + :type token: string + + :param user: the username to get the quota + :type user: string + + :param group: the group to get the quota for + :type group: string + + :param label: the label of an existing holding to get the details for + :type label: str, optional + + :param holding_id: the integer id of an existing holding to get the details + :type holding_id: int, optional + + :param tag: a list of key:value pairs to search holdings for - return + holdings with these tags. This will be converted to dictionary before + calling the remote method. + :type tag: dict, optional + + :raises requests.exceptions.ConnectionError: if the server cannot be + reached + + :return: A Dictionary of the response + :rtype: Dict + """ + # get the config, user and group + config = load_config() + token = load_token(config) + user = get_user(config, user) + group = get_group(config, group) + url = construct_server_url(config, "catalog/quota") + + # build the parameters. holdings->get requires + # user: str + # group: str + # token: str + input_params = {"user" : user, + "group" : group, + "token": token,} + + # add additional / optional components to input params + if label is not None: + input_params["label"] = label + if tag is not None: + input_params["tag"] = tag_to_string(tag) + if holding_id is not None: + input_params["holding_id"] = holding_id + if transaction_id is not None: + input_params["transaction_id"] = transaction_id + + response_dict = main_loop( + url=url, + input_params=input_params, + method=requests.get + ) + + if not response_dict: + response_dict = { + "msg" : f"FIND files for user {user} and group {group} failed", + "success" : False + } + # mark as failed in RPC call + elif "details" in response_dict and "failure" in response_dict["details"]: + response_dict["success"] = False + + return response_dict + + def init_client( url: str = None, verify_certificates: bool = True, diff --git a/nlds_client/nlds_client.py b/nlds_client/nlds_client.py index b07eb48..f660249 100755 --- a/nlds_client/nlds_client.py +++ b/nlds_client/nlds_client.py @@ -16,7 +16,7 @@ list_holding, find_file, monitor_transactions, get_transaction_state, - change_metadata, + change_metadata, get_quota, init_client) from nlds_client.clientlib.exceptions import (ConnectionError, RequestError, @@ -750,6 +750,55 @@ def find(user, group, groupall, label, holding_id, transaction_id, path, tag, except RequestError as re: raise click.UsageError(re) +"""Quota command""" +@nlds_client.command("quota", + help=f"Get the quota for a particular service.{user_help_text}") +@click.option("-T", "--token", default=None, type=str, + help="The token used to get the quota information.") +@click.option("-u", "--user", default=None, type=str, + help="The username of the user getting the quota.") +@click.option("-g", "--group", default=None, type=str, + help="The group to get the quota for.") +@click.option("-l", "--label", default=None, type=str, + help="The label of the holding which the files belong to. This " + "can be a regular expression (regex).") +@click.option("-i", "--holding_id", default=None, type=int, + help="The numeric id of the holding which the files belong to.") +@click.option("-n", "--transaction_id", default=None, type=str, + help="The UUID transaction id of the transaction to list.") +@click.option("-t", "--tag", default=None, type=TagParamType(), + help="The tag(s) of the holding(s) to find files within.") +@click.option("-j", "--json", default=False, type=bool, is_flag=True, + help="Output the result as JSON.") + +def quota(token, user, group, label, holding_id, transaction_id, tag, + json): + # + try: + response = get_quota(token, user, group, label, holding_id, transaction_id, tag) + req_details = format_request_details( + user, group, label=label, + holding_id=holding_id, tag=tag, transaction_id=transaction_id + ) + if response['success']: + if json: + click.echo(json_dumps(response)) + else: + print_find(response, req_details) + else: + fail_string = "Failed to get quota with " + fail_string += req_details + if response['details']['failure']: + fail_string += "\n" + response['details']['failure'] + raise click.UsageError(fail_string) + + except ConnectionError as ce: + raise click.UsageError(ce) + except AuthenticationError as ae: + raise click.UsageError(ae) + except RequestError as re: + raise click.UsageError(re) + """Meta command""" @nlds_client.command("meta", help=f"Alter metadata for a holding.{user_help_text}") From 854d237d3d9ff1382bd6a8dbbadd21ea0443664f Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Mon, 16 Dec 2024 13:59:06 +0000 Subject: [PATCH 2/7] Add quota command --- nlds_client/clientlib/transactions.py | 2 +- nlds_client/nlds_client.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/nlds_client/clientlib/transactions.py b/nlds_client/clientlib/transactions.py index 8a04387..c793024 100644 --- a/nlds_client/clientlib/transactions.py +++ b/nlds_client/clientlib/transactions.py @@ -1006,7 +1006,7 @@ def get_quota(token: str, if not response_dict: response_dict = { - "msg" : f"FIND files for user {user} and group {group} failed", + "msg" : f"GET QUOTA for user {user} and group {group} failed", "success" : False } # mark as failed in RPC call diff --git a/nlds_client/nlds_client.py b/nlds_client/nlds_client.py index f660249..4fc948f 100755 --- a/nlds_client/nlds_client.py +++ b/nlds_client/nlds_client.py @@ -357,6 +357,15 @@ def print_meta(response:dict, req_details:str): click.echo(f"{'':<12}{'label':<8}: {h['new_meta']['label']}") click.echo(f"{'':<12}{'tags':<8}: {h['new_meta']['tags']}") +def print_quota(response: dict, req_details:str): + """Print out the response from the client command""" + quota_string = "The allocated quota for the group " + quota_string += response['details']['group'] + quota_string += " is " + quota_string += str(response['data']['holdings']) + quota_string += " GB." + click.echo(quota_string) + def print_response(tr:dict): if 'msg' in tr and len(tr['msg']) > 0: @@ -784,7 +793,7 @@ def quota(token, user, group, label, holding_id, transaction_id, tag, if json: click.echo(json_dumps(response)) else: - print_find(response, req_details) + print_quota(response, req_details) else: fail_string = "Failed to get quota with " fail_string += req_details From 1b621b5fd213f5948c1ce9c705fd0ea3629e780f Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 18 Dec 2024 11:18:13 +0000 Subject: [PATCH 3/7] Remove uneccesary params --- nlds_client/clientlib/transactions.py | 31 ++-------------------- nlds_client/nlds_client.py | 38 +++++++++------------------ 2 files changed, 14 insertions(+), 55 deletions(-) diff --git a/nlds_client/clientlib/transactions.py b/nlds_client/clientlib/transactions.py index c793024..4c30bab 100644 --- a/nlds_client/clientlib/transactions.py +++ b/nlds_client/clientlib/transactions.py @@ -939,16 +939,10 @@ def change_metadata(user: str, return response_dict -def get_quota(token: str, - user: str, +def get_quota(user: str, group: str, - label: str=None, - holding_id: int=None, - transaction_id: str=None, - tag: dict=None): + ): """Make a request to get the quota for a particular group in the NLDS - :param token: the auth token needed to get the quota - :type token: string :param user: the username to get the quota :type user: string @@ -956,17 +950,6 @@ def get_quota(token: str, :param group: the group to get the quota for :type group: string - :param label: the label of an existing holding to get the details for - :type label: str, optional - - :param holding_id: the integer id of an existing holding to get the details - :type holding_id: int, optional - - :param tag: a list of key:value pairs to search holdings for - return - holdings with these tags. This will be converted to dictionary before - calling the remote method. - :type tag: dict, optional - :raises requests.exceptions.ConnectionError: if the server cannot be reached @@ -988,16 +971,6 @@ def get_quota(token: str, "group" : group, "token": token,} - # add additional / optional components to input params - if label is not None: - input_params["label"] = label - if tag is not None: - input_params["tag"] = tag_to_string(tag) - if holding_id is not None: - input_params["holding_id"] = holding_id - if transaction_id is not None: - input_params["transaction_id"] = transaction_id - response_dict = main_loop( url=url, input_params=input_params, diff --git a/nlds_client/nlds_client.py b/nlds_client/nlds_client.py index 4fc948f..9dbeaff 100755 --- a/nlds_client/nlds_client.py +++ b/nlds_client/nlds_client.py @@ -358,13 +358,15 @@ def print_meta(response:dict, req_details:str): click.echo(f"{'':<12}{'tags':<8}: {h['new_meta']['tags']}") def print_quota(response: dict, req_details:str): - """Print out the response from the client command""" - quota_string = "The allocated quota for the group " - quota_string += response['details']['group'] - quota_string += " is " - quota_string += str(response['data']['holdings']) - quota_string += " GB." - click.echo(quota_string) + """Print out the response from the quota command""" + try: + group = response['details']['group'] + quota = response['data']['holdings'] + quota_string = f"The allocated quota for the group {group} is {quota} GB." + click.echo(f"Checking quota for {req_details}:") + click.echo(quota_string) + except KeyError as e: + click.echo(f"Error: Missing key in response data - {e}") def print_response(tr:dict): @@ -762,32 +764,16 @@ def find(user, group, groupall, label, holding_id, transaction_id, path, tag, """Quota command""" @nlds_client.command("quota", help=f"Get the quota for a particular service.{user_help_text}") -@click.option("-T", "--token", default=None, type=str, - help="The token used to get the quota information.") @click.option("-u", "--user", default=None, type=str, help="The username of the user getting the quota.") @click.option("-g", "--group", default=None, type=str, help="The group to get the quota for.") -@click.option("-l", "--label", default=None, type=str, - help="The label of the holding which the files belong to. This " - "can be a regular expression (regex).") -@click.option("-i", "--holding_id", default=None, type=int, - help="The numeric id of the holding which the files belong to.") -@click.option("-n", "--transaction_id", default=None, type=str, - help="The UUID transaction id of the transaction to list.") -@click.option("-t", "--tag", default=None, type=TagParamType(), - help="The tag(s) of the holding(s) to find files within.") -@click.option("-j", "--json", default=False, type=bool, is_flag=True, - help="Output the result as JSON.") -def quota(token, user, group, label, holding_id, transaction_id, tag, - json): - # +def quota(user, group, ): try: - response = get_quota(token, user, group, label, holding_id, transaction_id, tag) + response = get_quota(user, group, ) req_details = format_request_details( - user, group, label=label, - holding_id=holding_id, tag=tag, transaction_id=transaction_id + user, group, ) if response['success']: if json: From 175fc3f3692885bc50393addabe10df48c309342 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 18 Dec 2024 11:45:34 +0000 Subject: [PATCH 4/7] Rename dict key for quota response --- nlds_client/nlds_client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nlds_client/nlds_client.py b/nlds_client/nlds_client.py index 9dbeaff..bb55d01 100755 --- a/nlds_client/nlds_client.py +++ b/nlds_client/nlds_client.py @@ -361,8 +361,11 @@ def print_quota(response: dict, req_details:str): """Print out the response from the quota command""" try: group = response['details']['group'] - quota = response['data']['holdings'] - quota_string = f"The allocated quota for the group {group} is {quota} GB." + quota = response['data']['quota'] + if quota == 0: + quota_string = f"There is no allocated quota for the group {group}." + else: + quota_string = f"The allocated quota for the group {group} is {quota} GB." click.echo(f"Checking quota for {req_details}:") click.echo(quota_string) except KeyError as e: From 4d88ef2bc7ce0128999941f3652926fcdb3dddc2 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Thu, 19 Dec 2024 10:50:13 +0000 Subject: [PATCH 5/7] Add quota command to client docs --- docs/user_guide/source/command_ref.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/user_guide/source/command_ref.rst b/docs/user_guide/source/command_ref.rst index 3e349e8..c5e1807 100644 --- a/docs/user_guide/source/command_ref.rst +++ b/docs/user_guide/source/command_ref.rst @@ -23,6 +23,7 @@ Commands: | ``put Put a single file.`` | ``putlist Put a number of files specified in a list.`` | ``stat List transactions.`` + ``quota Get the tape quota for the group.`` Each command has its own specific options. The argument is generally the file or filelist that the user wishes to operate on. The full command listing is From ae90e70205dd317050ab4ce7e035e457228922fc Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Thu, 27 Mar 2025 09:29:19 +0000 Subject: [PATCH 6/7] Add diskspace to quota command --- nlds_client/nlds_client.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/nlds_client/nlds_client.py b/nlds_client/nlds_client.py index bb55d01..d8dd156 100755 --- a/nlds_client/nlds_client.py +++ b/nlds_client/nlds_client.py @@ -358,19 +358,33 @@ def print_meta(response:dict, req_details:str): click.echo(f"{'':<12}{'tags':<8}: {h['new_meta']['tags']}") def print_quota(response: dict, req_details:str): - """Print out the response from the quota command""" + """Print out the quota response from the quota command""" try: group = response['details']['group'] quota = response['data']['quota'] if quota == 0: - quota_string = f"There is no allocated quota for the group {group}." + quota_string = f" There is no allocated quota for the group {group}." else: - quota_string = f"The allocated quota for the group {group} is {quota} GB." + quota_string = f" The allocated quota for the group {group} is {quota} GB." click.echo(f"Checking quota for {req_details}:") click.echo(quota_string) except KeyError as e: click.echo(f"Error: Missing key in response data - {e}") +def print_diskspace(response: dict, req_details:str): + """Print out the diskspace response from the quota command""" + try: + group = response ['details']['group'] + diskspace = response['data']['diskspace'] + if diskspace == 0: + diskspace_string = f" There is no used diskspace for the group {group}." + else: + diskspace_string = f" The used diskspace for the group {group} is {diskspace} GB." + click.echo(f"Checking diskspace for {req_details}:") + click.echo(diskspace_string) + except KeyError as e: + click.echo(f"Error: Missing key in response data - {e}") + def print_response(tr:dict): if 'msg' in tr and len(tr['msg']) > 0: @@ -783,6 +797,7 @@ def quota(user, group, ): click.echo(json_dumps(response)) else: print_quota(response, req_details) + print_diskspace(response, req_details) else: fail_string = "Failed to get quota with " fail_string += req_details From 628022c16f32b865bf997639e1c512e41ab9fcc4 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 14 May 2025 15:04:59 +0100 Subject: [PATCH 7/7] Work out remaining quota --- nlds_client/nlds_client.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/nlds_client/nlds_client.py b/nlds_client/nlds_client.py index d8dd156..0320354 100755 --- a/nlds_client/nlds_client.py +++ b/nlds_client/nlds_client.py @@ -361,11 +361,11 @@ def print_quota(response: dict, req_details:str): """Print out the quota response from the quota command""" try: group = response['details']['group'] - quota = response['data']['quota'] + quota = pretty_size(response['data']['quota']) if quota == 0: quota_string = f" There is no allocated quota for the group {group}." else: - quota_string = f" The allocated quota for the group {group} is {quota} GB." + quota_string = f" The allocated quota for the group {group} is {quota}." click.echo(f"Checking quota for {req_details}:") click.echo(quota_string) except KeyError as e: @@ -374,17 +374,39 @@ def print_quota(response: dict, req_details:str): def print_diskspace(response: dict, req_details:str): """Print out the diskspace response from the quota command""" try: - group = response ['details']['group'] - diskspace = response['data']['diskspace'] + group = response['details']['group'] + diskspace = pretty_size(response['data']['diskspace']) if diskspace == 0: diskspace_string = f" There is no used diskspace for the group {group}." else: - diskspace_string = f" The used diskspace for the group {group} is {diskspace} GB." + diskspace_string = f" The used diskspace for the group {group} is {diskspace}." click.echo(f"Checking diskspace for {req_details}:") click.echo(diskspace_string) except KeyError as e: click.echo(f"Error: Missing key in response data - {e}") +def print_remaining_quota(response: dict, req_details:str): + """Print out the diskspace response from the quota command""" + try: + group = response['details']['group'] + quota = response['data']['quota'] + diskspace = response['data']['diskspace'] + + # Determine the appropriate message based on quota and diskspace + if quota == 0 and diskspace == 0: + return # Print nothing if there is no quota and no diskspace + elif quota == 0 and diskspace > 0: + message = f"{pretty_size(diskspace)} over quota." + elif quota >= diskspace: + message = f"{pretty_size(quota - diskspace)} remaining." + else: + message = f"{pretty_size(diskspace-quota)} over quota." + + click.echo(" ") + click.echo(message) + except KeyError as e: + click.echo(f"Error: Missing key in response data - {e}") + def print_response(tr:dict): if 'msg' in tr and len(tr['msg']) > 0: @@ -798,6 +820,7 @@ def quota(user, group, ): else: print_quota(response, req_details) print_diskspace(response, req_details) + print_remaining_quota(response, req_details) else: fail_string = "Failed to get quota with " fail_string += req_details