From 34eeb08e4208e01dac2fd01a349bcdddd5f3e565 Mon Sep 17 00:00:00 2001 From: Shobhit Singh Date: Fri, 19 Jan 2024 01:24:36 +0000 Subject: [PATCH 1/2] chore: Script to inspect and clean up stale GCFs --- scripts/manage_cloud_functions.py | 195 ++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 scripts/manage_cloud_functions.py diff --git a/scripts/manage_cloud_functions.py b/scripts/manage_cloud_functions.py new file mode 100644 index 0000000000..5a6ed1bfe1 --- /dev/null +++ b/scripts/manage_cloud_functions.py @@ -0,0 +1,195 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from datetime import datetime +import sys +import time + +import google.api_core.exceptions +from google.cloud import functions_v2 + +GCF_REGIONS_ALL = [ + "asia-east1", + "asia-east2", + "asia-northeast1", + "asia-northeast2", + "europe-north1", + "europe-southwest1", + "europe-west1", + "europe-west2", + "europe-west4", + "europe-west8", + "europe-west9", + "us-central1", + "us-east1", + "us-east4", + "us-east5", + "us-south1", + "us-west1", + "asia-east2", + "asia-northeast3", + "asia-southeast1", + "asia-southeast2", + "asia-south1", + "asia-south2", + "australia-southeast1", + "australia-southeast2", + "europe-central2", + "europe-west2", + "europe-west3", + "europe-west6", + "northamerica-northeast1", + "northamerica-northeast2", + "southamerica-east1", + "southamerica-west1", + "us-west2", + "us-west3", + "us-west4", +] + +GCF_CLIENT = functions_v2.FunctionServiceClient() + + +def get_bigframes_functions(project, region): + parent = f"projects/{args.project_id}/locations/{region}" + functions = GCF_CLIENT.list_functions( + functions_v2.ListFunctionsRequest(parent=parent) + ) + # Filter bigframes created functions + functions = [ + function + for function in functions + if function.name.startswith( + f"projects/{args.project_id}/locations/{region}/functions/bigframes-" + ) + ] + + return functions + + +def summarize_gcfs(args): + """Summarize number of bigframes cloud functions in various regions.""" + + region_counts = {} + for region in args.regions: + functions = get_bigframes_functions(args.project_id, region) + functions_count = len(functions) + + # Exclude reporting regions with 0 bigframes GCFs + if functions_count == 0: + continue + + # Count how many GCFs are newer than a day + recent = 0 + for f in functions: + age = datetime.now() - datetime.fromtimestamp(f.update_time.timestamp()) + if age.days <= 0: + recent += 1 + + region_counts[region] = (functions_count, recent) + + for item in sorted( + region_counts.items(), key=lambda item: item[1][0], reverse=True + ): + region = item[0] + count, recent = item[1] + print( + "{}: Total={}, Recent={}, OlderThanADay={}".format( + region, count, recent, count - recent + ) + ) + + +def cleanup_gcfs(args): + """Clean-up bigframes cloud functions in the given regions.""" + max_delete_per_region = args.number + + for region in args.regions: + functions = get_bigframes_functions(args.project_id, region) + count = 0 + for f in functions: + age = datetime.now() - datetime.fromtimestamp(f.update_time.timestamp()) + if age.days > 0: + try: + count += 1 + GCF_CLIENT.delete_function(name=f.name) + print( + f"[{count}]: deleted [{count}] {f.name} last updated on {f.update_time}" + ) + if count >= max_delete_per_region: + break + # Mostly there is a 60 mutations per minute quota, we want to use 10% of + # that for this clean-up, i.e. 6 mutations per minute. So wait for + # 60/6 = 10 seconds + time.sleep(10) + except google.api_core.exceptions.ResourceExhausted: + # Stop deleting in this region for now + print( + "Cannot delete any more functions in region {region} due to quota exhaustion. Please try again later." + ) + break + + +def list_str(values): + return [val for val in values.split(",") if val] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Manage cloud functions created to serve bigframes remote functions." + ) + parser.add_argument( + "-p", + "--project-id", + type=str, + required=True, + action="store", + help="GCP project-id.", + ) + parser.add_argument( + "-r", + "--regions", + type=list_str, + required=False, + default=GCF_REGIONS_ALL, + action="store", + help="Cloud functions region(s). If multiple regions, Specify comma separated (e.g. region1,region2)", + ) + + subparsers = parser.add_subparsers(title="subcommands", required=True) + parser_summary = subparsers.add_parser( + "summary", + help="BigFrames cloud functions summary.", + description="Show the bigframes cloud functions summary.", + ) + parser_summary.set_defaults(func=summarize_gcfs) + parser_cleanup = subparsers.add_parser( + "cleanup", + help="BigFrames cloud functions clean up.", + description="Delete the stale bigframes cloud functions.", + ) + parser_cleanup.add_argument( + "-n", + "--number", + type=int, + required=False, + default=100, + action="store", + help="Number of stale (more than a day old) cloud functions to clean up.", + ) + parser_cleanup.set_defaults(func=cleanup_gcfs) + + args = parser.parse_args(sys.argv[1:]) + args.func(args) From fa375b7598284f22814a4392e6fc79673a238ee3 Mon Sep 17 00:00:00 2001 From: Shobhit Singh Date: Fri, 19 Jan 2024 01:35:44 +0000 Subject: [PATCH 2/2] fix formatting --- scripts/manage_cloud_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/manage_cloud_functions.py b/scripts/manage_cloud_functions.py index 5a6ed1bfe1..6b69089089 100644 --- a/scripts/manage_cloud_functions.py +++ b/scripts/manage_cloud_functions.py @@ -126,7 +126,7 @@ def cleanup_gcfs(args): count += 1 GCF_CLIENT.delete_function(name=f.name) print( - f"[{count}]: deleted [{count}] {f.name} last updated on {f.update_time}" + f"[{region}]: deleted [{count}] {f.name} last updated on {f.update_time}" ) if count >= max_delete_per_region: break @@ -137,7 +137,7 @@ def cleanup_gcfs(args): except google.api_core.exceptions.ResourceExhausted: # Stop deleting in this region for now print( - "Cannot delete any more functions in region {region} due to quota exhaustion. Please try again later." + f"Cannot delete any more functions in region {region} due to quota exhaustion. Please try again later." ) break