From a04480572903a5bfc7b85cf64eea2f1ac998b010 Mon Sep 17 00:00:00 2001 From: sressier Date: Thu, 18 Sep 2025 14:27:30 +0200 Subject: [PATCH 1/9] new module --- .../iroco2-client-side-scanner/README.md | 5 +- .../cur_scrapper.py | 62 +++ .../python3.11/site-packages/ec2_service.py | 32 +- .../python3.11/site-packages/iroco_service.py | 14 +- .../site-packages/s3_repository_parser.py | 24 +- .../python3.11/site-packages/s3_service.py | 30 +- .../package/cur_scrapper.py | 32 +- .../python/ec2_service.py | 99 ++++ .../python/iroco_service.py | 67 +++ .../python/s3_repository_parser.py | 27 ++ .../python/s3_service.py | 116 +++++ .../iroco2-client-side-scanner/tf/.gitignore | 22 + .../tf/.pre-commit-config.yaml | 38 ++ .../iroco2-client-side-scanner/tf/.tflint.hcl | 95 ++++ .../iroco2-client-side-scanner/tf/README.md | 452 ++++++++++++++++++ .../tf/cloudwatch.tf | 8 + .../tf/examples/basic/README.md | 121 +++++ .../tf/examples/basic/main.tf | 89 ++++ .../examples/basic/terraform.tfvars.example | 23 + .../tf/examples/basic/variables.tf | 67 +++ .../iroco2-client-side-scanner/tf/iam.tf | 52 ++ .../iroco2-client-side-scanner/tf/lambda.tf | 67 +++ .../iroco2-client-side-scanner/tf/locals.tf | 30 ++ .../iroco2-client-side-scanner/tf/main.tf | 5 + .../iroco2-client-side-scanner/tf/outputs.tf | 59 +++ .../tf/providers.tf | 11 + .../tf/response.json | 1 + .../tf/response2.json | 1 + .../tf/response3.json | 1 + .../tf/response4.json | 1 + .../tf/response5.json | 1 + .../iroco2-client-side-scanner/tf/s3.tf | 97 ++++ .../tf/terraform.tfvars.example | 29 ++ .../tf/test-payload.json | 15 + .../tf/variables.tf | 128 +++++ .../iroco2-client-side-scanner/tf/versions.tf | 10 + 36 files changed, 1861 insertions(+), 70 deletions(-) create mode 100644 lambdas/functions/iroco2-client-side-scanner/cur_scrapper.py create mode 100644 lambdas/functions/iroco2-client-side-scanner/python/ec2_service.py create mode 100644 lambdas/functions/iroco2-client-side-scanner/python/iroco_service.py create mode 100644 lambdas/functions/iroco2-client-side-scanner/python/s3_repository_parser.py create mode 100644 lambdas/functions/iroco2-client-side-scanner/python/s3_service.py create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/.gitignore create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/.pre-commit-config.yaml create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/.tflint.hcl create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/README.md create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/cloudwatch.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/README.md create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/main.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/terraform.tfvars.example create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/variables.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/iam.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/lambda.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/locals.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/main.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/outputs.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/providers.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/response.json create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/response2.json create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/response3.json create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/response4.json create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/response5.json create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/s3.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/terraform.tfvars.example create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/test-payload.json create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/variables.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/versions.tf diff --git a/lambdas/functions/iroco2-client-side-scanner/README.md b/lambdas/functions/iroco2-client-side-scanner/README.md index 937c106..8256aa0 100644 --- a/lambdas/functions/iroco2-client-side-scanner/README.md +++ b/lambdas/functions/iroco2-client-side-scanner/README.md @@ -8,7 +8,4 @@ This lambda is used to scan aws accounts to retrieve the CUR data and carbon imp - Upate docs to self host assets - - -SaaS mode documentation (french): https://docs.greensuite.ippon.fr/#/docs/guide_scanner - +SaaS mode documentation (french): diff --git a/lambdas/functions/iroco2-client-side-scanner/cur_scrapper.py b/lambdas/functions/iroco2-client-side-scanner/cur_scrapper.py new file mode 100644 index 0000000..91bbf1b --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/cur_scrapper.py @@ -0,0 +1,62 @@ +# Copyright 2025 Ippon Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +import os + +from s3_repository_parser import S3Repository +from ec2_service import CurProcessorEC2Service +from s3_service import CurProcessorS3Service +from iroco_service import IrocoService + +class CurProcessorService: + + def __init__(self): + self.s3_repository = S3Repository() + self.cur_processor_ec2_service = CurProcessorEC2Service() + self.cur_processor_s3_service = CurProcessorS3Service() + self.iroco_service = IrocoService() + + def parsing_cur_and_send_to_api(self, bucket_name, s3_file_name): + cur_file_type = os.path.splitext(s3_file_name)[1].replace(".", "") + print(cur_file_type) + cur = self.s3_repository.read_file(bucket_name, s3_file_name) + message_parsed = [] + parsed_ec2_message = self.cur_processor_ec2_service.creating_message_from_cur(cur, cur_file_type) + print(f'EC2 messages: {parsed_ec2_message}') + parsed_s3_message = self.cur_processor_s3_service.creating_message_from_cur(cur, cur_file_type) + print(f'S3 messages: {parsed_s3_message}') + message_parsed.extend(parsed_ec2_message) + message_parsed.extend(parsed_s3_message) + self.__send_message_parsed_to_api(message_parsed) + + def __send_message_parsed_to_api(self, messages_parsed): + total_message_sent = len(messages_parsed) + correlation_id = self.iroco_service.create_scan() + + for message_parsed in messages_parsed: + message_parsed["numberOfMessageExpected"] = total_message_sent + message_parsed["correlationId"] = correlation_id + self.iroco_service.send_payload(message_parsed) + +def handler(event,context): + print(f"Event Name: {event['Records'][0]['eventName']}") + print(f"S3 Bucket Name: {event['Records'][0]['s3']['bucket']['name']}") + print(f"Object Key: {event['Records'][0]['s3']['object']['key']}") + print(f"Event: {event}") + cur_processor_service = CurProcessorService() + cur_processor_service.parsing_cur_and_send_to_api( + bucket_name=event['Records'][0]['s3']['bucket']['name'], + s3_file_name=event['Records'][0]['s3']['object']['key'], + ) diff --git a/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/ec2_service.py b/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/ec2_service.py index f4ffd10..7d58560 100644 --- a/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/ec2_service.py +++ b/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/ec2_service.py @@ -1,18 +1,18 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 +# Copyright 2025 Ippon Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 import pandas as pd from io import StringIO, BytesIO import isodate @@ -96,4 +96,4 @@ def __create_parsed_data_message(self, row, cur_file_type): "serviceTypeCUR": service_type_cur } - return message \ No newline at end of file + return message diff --git a/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/iroco_service.py b/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/iroco_service.py index f873b52..987c350 100644 --- a/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/iroco_service.py +++ b/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/iroco_service.py @@ -22,7 +22,7 @@ def __init__(self): self.api_key = os.environ.get("IROCO2_API_KEY") self.iroco_endpoint = os.environ.get("IROCO2_API_ENDPOINT") self.gateway_endpoint = os.environ.get("IROCO2_GATEWAY_ENDPOINT") - + def __build_header(self): """ Build header @@ -53,15 +53,15 @@ def send_payload(self, payload): send_report = requests.post(self.gateway_endpoint, json=payload, headers=self.__build_header()) send_report.raise_for_status() return send_report.status_code - except requests.exceptions.HTTPError as errh: - print("HTTP Error") + except requests.exceptions.HTTPError as errh: + print("HTTP Error") print(errh.args[0]) - return send_report.status_code - except requests.exceptions.ConnectionError as conerr: + return send_report.status_code + except requests.exceptions.ConnectionError as conerr: print("Connection error") print(conerr) return send_report.status_code - except requests.exceptions.RequestException as errex: + except requests.exceptions.RequestException as errex: print("Exception request") print(errex) - return send_report.status_code \ No newline at end of file + return send_report.status_code diff --git a/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/s3_repository_parser.py b/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/s3_repository_parser.py index 0d2e005..30ae5c3 100644 --- a/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/s3_repository_parser.py +++ b/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/s3_repository_parser.py @@ -13,15 +13,15 @@ # limitations under the License. # # SPDX-License-Identifier: Apache-2.0 -import boto3 - -class S3Repository: - - def __init__(self): - self.s3_ressource = boto3.resource('s3') - - def read_file(self, bucket_name: str, key_to_read: str): - obj = self.s3_ressource.Object(bucket_name, key_to_read) - response = obj.get() - raw_content = response['Body'].read() - return raw_content +import boto3 + +class S3Repository: + + def __init__(self): + self.s3_ressource = boto3.resource('s3') + + def read_file(self, bucket_name: str, key_to_read: str): + obj = self.s3_ressource.Object(bucket_name, key_to_read) + response = obj.get() + raw_content = response['Body'].read() + return raw_content diff --git a/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/s3_service.py b/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/s3_service.py index c39607d..48476d1 100644 --- a/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/s3_service.py +++ b/lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/s3_service.py @@ -1,18 +1,18 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 +# Copyright 2025 Ippon Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 import pandas as pd import isodate diff --git a/lambdas/functions/iroco2-client-side-scanner/package/cur_scrapper.py b/lambdas/functions/iroco2-client-side-scanner/package/cur_scrapper.py index 529fb14..91bbf1b 100644 --- a/lambdas/functions/iroco2-client-side-scanner/package/cur_scrapper.py +++ b/lambdas/functions/iroco2-client-side-scanner/package/cur_scrapper.py @@ -1,18 +1,18 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 +# Copyright 2025 Ippon Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 import os from s3_repository_parser import S3Repository @@ -59,4 +59,4 @@ def handler(event,context): cur_processor_service.parsing_cur_and_send_to_api( bucket_name=event['Records'][0]['s3']['bucket']['name'], s3_file_name=event['Records'][0]['s3']['object']['key'], - ) \ No newline at end of file + ) diff --git a/lambdas/functions/iroco2-client-side-scanner/python/ec2_service.py b/lambdas/functions/iroco2-client-side-scanner/python/ec2_service.py new file mode 100644 index 0000000..7d58560 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/python/ec2_service.py @@ -0,0 +1,99 @@ +# Copyright 2025 Ippon Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +import pandas as pd +from io import StringIO, BytesIO +import isodate +from datetime import timedelta + +class CurProcessorEC2Service: + + cur_useful_column_keyname_matrix = { + 'service_code': { + 'zip': 'product/servicecode', + 'csv': 'product/servicecode', + 'parquet': 'product_servicecode' + }, + 'usage_type': { + 'zip': 'lineItem/UsageType', + 'csv': 'lineItem/UsageType', + 'parquet': 'line_item_usage_type' + }, + 'region_code': { + 'zip': 'product/regionCode', + 'csv': 'product/regionCode', + 'parquet': 'product_region_code' + }, + 'instance_type': { + 'zip': 'product/instanceType', + 'csv': 'product/instanceType', + 'parquet': 'product_instance_type' + } + } + + def creating_message_from_cur(self, cur, cur_file_type): + try: + useful_column = [values[cur_file_type] for values in self.cur_useful_column_keyname_matrix.values()] + match cur_file_type: + case 'zip': + cur_with_useful_colum = pd.read_csv(BytesIO(cur), usecols=useful_column, compression='zip', low_memory=False) + case 'csv': + cur_with_useful_colum = pd.read_csv(StringIO(cur.decode('utf-8')), usecols=useful_column, low_memory=False) + case 'parquet': + cur_with_useful_colum = pd.read_parquet(BytesIO(cur), columns=useful_column) + case _: + raise Exception(f'Unknown file type {cur_file_type}') + parsed_cur_by_service_ec2 = cur_with_useful_colum[ + cur_with_useful_colum[ + self.cur_useful_column_keyname_matrix['service_code'][cur_file_type] + ].isin(['AmazonEC2'])] + parsed_cur_by_service_ec2_and_box_usage = parsed_cur_by_service_ec2[ + parsed_cur_by_service_ec2[ + self.cur_useful_column_keyname_matrix['usage_type'][cur_file_type] + ].str.contains('BoxUsage', na=False)] + parsed_cur_ec2_grouped_by_instance_and_region = parsed_cur_by_service_ec2_and_box_usage.groupby( + [ + self.cur_useful_column_keyname_matrix['region_code'][cur_file_type], + self.cur_useful_column_keyname_matrix['instance_type'][cur_file_type] + ]).size().reset_index(name='numberOfLine') + + queue_messages = [] + for index, row in parsed_cur_ec2_grouped_by_instance_and_region.iterrows(): + queue_message = self.__create_parsed_data_message(row, cur_file_type) + queue_messages.append(queue_message) + + return queue_messages + except ValueError as e: + if 'Usecols do not match columns' in str(e): + print(f"ValueError: {e}") + return [] + else: + raise e + + def __create_parsed_data_message(self, row, cur_file_type): + aws_data_center = row[self.cur_useful_column_keyname_matrix['region_code'][cur_file_type]] + instance_type = row[self.cur_useful_column_keyname_matrix['instance_type'][cur_file_type]] + number_of_line = row['numberOfLine'] + duration = isodate.duration_isoformat(timedelta(hours=number_of_line)) + service_type_cur = "EC2" + + message = { + "ec2Type": instance_type, + "awsDataCenter": aws_data_center, + "durationOfServiceOperation": duration, + "serviceTypeCUR": service_type_cur + } + + return message diff --git a/lambdas/functions/iroco2-client-side-scanner/python/iroco_service.py b/lambdas/functions/iroco2-client-side-scanner/python/iroco_service.py new file mode 100644 index 0000000..987c350 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/python/iroco_service.py @@ -0,0 +1,67 @@ +# Copyright 2025 Ippon Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +import os +import requests + +class IrocoService: + + def __init__(self): + self.api_key = os.environ.get("IROCO2_API_KEY") + self.iroco_endpoint = os.environ.get("IROCO2_API_ENDPOINT") + self.gateway_endpoint = os.environ.get("IROCO2_GATEWAY_ENDPOINT") + + def __build_header(self): + """ + Build header + """ + return { + 'Authorization': f"Bearer {self.api_key}", + 'Content-Type': "application/json", + } + + def create_scan(self): + """ + Create a scan + """ + try: + response = requests.post(self.iroco_endpoint, headers=self.__build_header()) + response.raise_for_status() + except requests.exceptions.HTTPError as errh: + print("HTTP Error") + print(errh) + return None + return response.json() + + def send_payload(self, payload): + """ + Send a payload + """ + try: + send_report = requests.post(self.gateway_endpoint, json=payload, headers=self.__build_header()) + send_report.raise_for_status() + return send_report.status_code + except requests.exceptions.HTTPError as errh: + print("HTTP Error") + print(errh.args[0]) + return send_report.status_code + except requests.exceptions.ConnectionError as conerr: + print("Connection error") + print(conerr) + return send_report.status_code + except requests.exceptions.RequestException as errex: + print("Exception request") + print(errex) + return send_report.status_code diff --git a/lambdas/functions/iroco2-client-side-scanner/python/s3_repository_parser.py b/lambdas/functions/iroco2-client-side-scanner/python/s3_repository_parser.py new file mode 100644 index 0000000..30ae5c3 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/python/s3_repository_parser.py @@ -0,0 +1,27 @@ +# Copyright 2025 Ippon Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +import boto3 + +class S3Repository: + + def __init__(self): + self.s3_ressource = boto3.resource('s3') + + def read_file(self, bucket_name: str, key_to_read: str): + obj = self.s3_ressource.Object(bucket_name, key_to_read) + response = obj.get() + raw_content = response['Body'].read() + return raw_content diff --git a/lambdas/functions/iroco2-client-side-scanner/python/s3_service.py b/lambdas/functions/iroco2-client-side-scanner/python/s3_service.py new file mode 100644 index 0000000..48476d1 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/python/s3_service.py @@ -0,0 +1,116 @@ +# Copyright 2025 Ippon Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +import pandas as pd +import isodate + +from io import StringIO, BytesIO +from datetime import timedelta + + +class CurProcessorS3Service: + + cur_useful_column_keyname_matrix = { + 'service_code': { + 'zip': 'product/servicecode', + 'csv': 'product/servicecode', + 'parquet': 'product_servicecode' + }, + 'usage_type': { + 'zip': 'lineItem/UsageType', + 'csv': 'lineItem/UsageType', + 'parquet': 'line_item_usage_type' + }, + 'region_code': { + 'zip': 'product/regionCode', + 'csv': 'product/regionCode', + 'parquet': 'product_region_code' + }, + 'usage_amount': { + 'zip': 'lineItem/UsageAmount', + 'csv': 'lineItem/UsageAmount', + 'parquet': 'line_item_usage_amount' + } + } + + cur_file_type = None + + def creating_message_from_cur(self, cur, cur_file_type): + try: + self.cur_file_type = cur_file_type + queue_messages = [] + storage_queue_messages = self.__creating_message_from_cur_for_storage(cur) + queue_messages.extend(storage_queue_messages) + + return queue_messages + except ValueError as e: + if 'Usecols do not match columns' in str(e): + print(f"ValueError: {e}") + return [] + else: + raise e + + def __create_parsed_data_message(self, row): + aws_data_center = row[ + self.cur_useful_column_keyname_matrix['region_code'][self.cur_file_type] + ] + number_of_line = row['numberOfLines'] + total_usage_amount = row['totalUsageAmount'] + average_usage_amount = total_usage_amount / number_of_line + duration = isodate.duration_isoformat(timedelta(hours=number_of_line)) + service_type_cur = "S3" + + message = { + "storageInMo": average_usage_amount, + "awsDataCenter": aws_data_center, + "durationOfServiceOperation": duration, + "serviceTypeCUR": service_type_cur + } + + return message + + def __creating_message_from_cur_for_storage(self, cur): + useful_columns = [values[self.cur_file_type] for values in self.cur_useful_column_keyname_matrix.values()] + match self.cur_file_type: + case 'zip': + cur_with_useful_colum = pd.read_csv(BytesIO(cur), usecols=useful_columns, compression='zip', low_memory=False) + case 'csv': + cur_with_useful_colum = pd.read_csv(StringIO(cur.decode('utf-8')), usecols=useful_columns, low_memory=False) + case 'parquet': + cur_with_useful_colum = pd.read_parquet(BytesIO(cur), columns=useful_columns) + case _: + raise Exception(f'Unknown file type {self.cur_file_type}') + parsed_cur_by_service_s3 = ( + cur_with_useful_colum)[cur_with_useful_colum[ + self.cur_useful_column_keyname_matrix['service_code'][self.cur_file_type]] == 'AmazonS3' + ] + parsed_cur_by_service_s3_and_usage_type = parsed_cur_by_service_s3[ + parsed_cur_by_service_s3[ + self.cur_useful_column_keyname_matrix['usage_type'][self.cur_file_type] + ].str.contains('TimedStorage-ByteHrs', na=False)] + + parsed_cur_s3_grouped_by_instance_and_region = parsed_cur_by_service_s3_and_usage_type.groupby( + [self.cur_useful_column_keyname_matrix['region_code'][self.cur_file_type]] + ).agg( + totalUsageAmount=(self.cur_useful_column_keyname_matrix['usage_amount'][self.cur_file_type], 'sum'), + numberOfLines=(self.cur_useful_column_keyname_matrix['usage_amount'][self.cur_file_type], 'size') + ).reset_index() + + queue_messages = [] + for index, row in parsed_cur_s3_grouped_by_instance_and_region.iterrows(): + queue_message = self.__create_parsed_data_message(row) + queue_messages.append(queue_message) + + return queue_messages diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/.gitignore b/lambdas/functions/iroco2-client-side-scanner/tf/.gitignore new file mode 100644 index 0000000..fe5ac31 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/.gitignore @@ -0,0 +1,22 @@ +# Terraform files +*.tfstate +*.tfstate.* +*.tfvars +!terraform.tfvars.example +.terraform/ +.terraform.lock.hcl +crash.log +crash.*.log + +# IDE files +.vscode/ +.idea/ +*.swp +*.swo + +# OS files +.DS_Store +Thumbs.db + +# Pre-commit +.secrets.baseline diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/.pre-commit-config.yaml b/lambdas/functions/iroco2-client-side-scanner/tf/.pre-commit-config.yaml new file mode 100644 index 0000000..3721e0a --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/.pre-commit-config.yaml @@ -0,0 +1,38 @@ +# Simplified pre-commit configuration for IROCO2 Client Side Scanner Terraform module +# This configuration includes only essential checks that work without additional tool installation + +repos: + # General hooks + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + + # Terraform hooks (minimal set) + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.88.0 + hooks: + - id: terraform_fmt + description: Rewrites all Terraform configuration files to a canonical format + - id: terraform_validate + description: Validates all Terraform configuration files + args: + - --hook-config=--retry-once-with-cleanup=true + +# Configuration for specific hooks +default_language_version: + python: python3 + +# Global excludes +exclude: | + (?x)^( + \.terraform/.*| + \.terraform\.lock\.hcl| + terraform\.tfstate.*| + \.git/.* + )$ diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/.tflint.hcl b/lambdas/functions/iroco2-client-side-scanner/tf/.tflint.hcl new file mode 100644 index 0000000..924d8ff --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/.tflint.hcl @@ -0,0 +1,95 @@ +# TFLint configuration for IROCO2 Client Side Scanner Terraform module + +config { + # Enable all rules by default + disabled_by_default = false + + # Plugin directory + plugin_dir = "~/.tflint.d/plugins" + + # Call module inspection + call_module_type = "all" +} + +# AWS plugin configuration +plugin "aws" { + enabled = true + version = "0.29.0" + source = "github.com/terraform-linters/tflint-ruleset-aws" + + # Deep checking requires AWS credentials + deep_check = false +} + +# Terraform plugin configuration +plugin "terraform" { + enabled = true + preset = "recommended" +} + +# Rule configurations +rule "terraform_deprecated_interpolation" { + enabled = true +} + +rule "terraform_deprecated_index" { + enabled = true +} + +rule "terraform_unused_declarations" { + enabled = true +} + +rule "terraform_comment_syntax" { + enabled = true +} + +rule "terraform_documented_outputs" { + enabled = true +} + +rule "terraform_documented_variables" { + enabled = true +} + +rule "terraform_typed_variables" { + enabled = true +} + +rule "terraform_module_pinned_source" { + enabled = true +} + +rule "terraform_naming_convention" { + enabled = true + format = "snake_case" +} + +rule "terraform_required_version" { + enabled = true +} + +rule "terraform_required_providers" { + enabled = true +} + +rule "terraform_standard_module_structure" { + enabled = true +} + +# AWS specific rules +rule "aws_instance_invalid_type" { + enabled = true +} + +rule "aws_lambda_function_invalid_runtime" { + enabled = true +} + +rule "aws_s3_bucket_invalid_region" { + enabled = true +} + +rule "aws_iam_policy_invalid_policy" { + enabled = true +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/README.md b/lambdas/functions/iroco2-client-side-scanner/tf/README.md new file mode 100644 index 0000000..bca5eaa --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/README.md @@ -0,0 +1,452 @@ +# IROCO2 Client Side Scanner Terraform Module + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Terraform](https://img.shields.io/badge/Terraform-%3E%3D1.0-623CE4)](https://www.terraform.io/) +[![AWS](https://img.shields.io/badge/AWS-%3E%3D5.0-FF9900)](https://aws.amazon.com/) + +This Terraform module creates the AWS infrastructure for the IROCO2 Client Side Scanner, which processes Cost and Usage Report (CUR) data for carbon impact analysis. + +> **Open Source Ready**: This module is designed for easy integration into open source projects and can be used by anyone to analyze AWS carbon footprint data. + +## Quick Start + +Want to get started quickly? Check out our [basic example](examples/basic/) which includes: + +- πŸ“‹ **Step-by-step setup guide** +- πŸ”§ **Pre-configured variables** +- πŸ“– **Detailed documentation** +- πŸš€ **Ready-to-deploy configuration** + +```bash +# Clone the example +cp -r examples/basic my-iroco2-deployment +cd my-iroco2-deployment + +# Configure your settings +cp terraform.tfvars.example terraform.tfvars +# Edit terraform.tfvars with your AWS details + +# Deploy +terraform init && terraform apply +``` + +## Overview + +The module provisions the following AWS resources: + +- **S3 Bucket**: Encrypted storage for CUR output with event notifications +- **Lambda Function**: Python 3.11 runtime for processing CUR data +- **Lambda Layer**: Custom helper scripts for IROCO2 functionality +- **IAM Role & Policies**: Execution permissions for Lambda with least privilege access +- **CloudWatch Log Group**: Centralized logging with KMS encryption +- **S3 Bucket Policy**: Organization-based access control +- **Lambda Permissions**: S3 trigger configuration + +## Architecture + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ S3 Bucket │───▢│ Lambda Function │───▢│ IROCO2 API β”‚ +β”‚ (CUR Output) β”‚ β”‚ (CUR Scanner) β”‚ β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β”‚ β–Ό + β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ β”‚ CloudWatch Logs β”‚ + β”‚ β”‚ β”‚ + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ KMS Key β”‚ +β”‚ (Encryption) β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +## Usage + +### Basic Usage + +```hcl +module "iroco2_client_side_scanner" { + source = "./lambdas/functions/iroco2-client-side-scanner/tf" + + # Required variables + kms_key_arn = "arn:aws:kms:eu-west-3:123456789012:key/12345678-1234-1234-1234-123456789012" + layer_bucket_storage = "my-lambda-layers-bucket" + layer_bucket_key = "layers/helper-scripts.zip" + aws_org_id = "o-1234567890" + cur_output_bucket_name = "my-cur-output-bucket" + cur_function_s3_key = "functions/cur-scanner.zip" + cur_function_s3_bucket = "my-lambda-functions-bucket" + iroco2_api_endpoint = "https://api.iroco2.example.com" + iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" + iroco2_api_key = "your-api-key-here" + + # Optional variables with defaults + lambda_function_name = "IROCO2-CUR-SCRAPPER" + lambda_log_group_name = "/aws/lambda/IROCO2-CUR-SCRAPPER" + lambda_timeout = 900 + lambda_memory_size = 512 + log_retention_days = 14 + + common_tags = { + Environment = "production" + Project = "IROCO2" + Owner = "data-team" + } +} +``` + +### Advanced Usage with Custom Configuration + +```hcl +module "iroco2_client_side_scanner" { + source = "./lambdas/functions/iroco2-client-side-scanner/tf" + + # Required variables + kms_key_arn = data.aws_kms_key.main.arn + layer_bucket_storage = aws_s3_bucket.lambda_layers.bucket + layer_bucket_key = "layers/helper-scripts-v2.zip" + aws_org_id = var.organization_id + cur_output_bucket_name = "${var.environment}-cur-output-${random_id.bucket_suffix.hex}" + cur_function_s3_key = "functions/cur-scanner-${var.version}.zip" + cur_function_s3_bucket = aws_s3_bucket.lambda_functions.bucket + iroco2_api_endpoint = var.iroco2_api_endpoint + iroco2_gateway_endpoint = var.iroco2_gateway_endpoint + iroco2_api_key = data.aws_secretsmanager_secret_version.iroco2_api_key.secret_string + + # Custom configuration + lambda_function_name = "${var.environment}-iroco2-cur-scanner" + lambda_log_group_name = "/aws/lambda/${var.environment}-iroco2-cur-scanner" + lambda_timeout = 600 + lambda_memory_size = 1024 + log_retention_days = 30 + + common_tags = merge(var.common_tags, { + Environment = var.environment + Module = "iroco2-client-side-scanner" + }) +} +``` + +## Requirements + +| Name | Version | +|------|---------| +| terraform | >= 1.0 | +| aws | >= 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| aws | >= 5.0 | + +## Resources + +| Name | Type | +|------|------| +| aws_s3_bucket.cur_output | resource | +| aws_s3_bucket_versioning.cur_output | resource | +| aws_s3_bucket_server_side_encryption_configuration.cur_output | resource | +| aws_s3_bucket_public_access_block.cur_output | resource | +| aws_s3_bucket_policy.cur_output | resource | +| aws_s3_bucket_notification.cur_output | resource | +| aws_lambda_function.processing | resource | +| aws_lambda_layer_version.helper_scripts | resource | +| aws_lambda_permission.s3_invoke | resource | +| aws_iam_role.lambda_execution | resource | +| aws_iam_role_policy_attachment.lambda_basic_execution | resource | +| aws_iam_role_policy.lambda_s3_kms_access | resource | +| aws_cloudwatch_log_group.lambda | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| kms_key_arn | The KMS key ARN to encrypt the bucket and logs | `string` | n/a | yes | +| layer_bucket_storage | The S3 bucket name for the lambda layer | `string` | n/a | yes | +| layer_bucket_key | The S3 bucket key for the lambda layer | `string` | n/a | yes | +| aws_org_id | The AWS Organization ID | `string` | n/a | yes | +| cur_output_bucket_name | The S3 bucket name for the CUR output | `string` | n/a | yes | +| cur_function_s3_key | The S3 bucket key for the CUR lambda function | `string` | n/a | yes | +| cur_function_s3_bucket | The S3 bucket name for the CUR lambda function | `string` | n/a | yes | +| iroco2_api_endpoint | The IroCO2 API endpoint | `string` | n/a | yes | +| iroco2_gateway_endpoint | The IroCO2 Gateway endpoint | `string` | n/a | yes | +| iroco2_api_key | The IroCO2 API token | `string` | n/a | yes | +| lambda_function_name | Name of the Lambda function | `string` | `"IROCO2-CUR-SCRAPPER"` | no | +| lambda_log_group_name | Log group name for lambda | `string` | `"/aws/lambda/IROCO2-CUR-SCRAPPER"` | no | +| lambda_timeout | Lambda function timeout in seconds | `number` | `900` | no | +| lambda_memory_size | Lambda function memory size in MB | `number` | `512` | no | +| log_retention_days | CloudWatch log retention in days | `number` | `14` | no | +| common_tags | Common tags to apply to all resources | `map(string)` | `{"Project": "IROCO2"}` | no | +| aws_sdk_pandas_layer_arn | ARN of the AWS SDK Pandas layer | `string` | `"arn:aws:lambda:eu-west-3:336392948345:layer:AWSSDKPandas-Python311:18"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| lambda_function_name | Name of the created Lambda function for CUR extraction | +| lambda_function_arn | ARN of the created Lambda function | +| lambda_function_invoke_arn | Invoke ARN of the Lambda function | +| s3_bucket_name | Name of the S3 bucket for CUR output | +| s3_bucket_arn | ARN of the S3 bucket for CUR output | +| s3_bucket_domain_name | Domain name of the S3 bucket | +| lambda_execution_role_arn | ARN of the Lambda execution role | +| lambda_execution_role_name | Name of the Lambda execution role | +| lambda_layer_arn | ARN of the helper scripts Lambda layer | +| lambda_layer_version | Version of the helper scripts Lambda layer | +| cloudwatch_log_group_name | Name of the CloudWatch log group | +| cloudwatch_log_group_arn | ARN of the CloudWatch log group | + +## Features + +### Security + +- **KMS Encryption**: All data at rest is encrypted using customer-managed KMS keys +- **IAM Least Privilege**: Lambda execution role has minimal required permissions +- **Organization Access Control**: S3 bucket policy restricts access to organization members +- **Secure Transport**: All S3 operations require HTTPS + +### Monitoring & Logging + +- **CloudWatch Integration**: Centralized logging with configurable retention +- **Structured Logging**: Lambda function uses structured logging format +- **Error Tracking**: Failed executions are logged for troubleshooting + +### Scalability + +- **Configurable Resources**: Memory and timeout can be adjusted based on workload +- **Layer Management**: Helper scripts are packaged as reusable Lambda layers +- **Event-Driven**: S3 events automatically trigger processing + +## Triggers + +The Lambda function is automatically triggered by S3 events: + +- **Event Type**: `s3:ObjectCreated:Copy` +- **File Filters**: + - Files ending with `.parquet` + - Files ending with `.csv.zip` + +## Environment Variables + +The Lambda function receives the following environment variables: + +- `IROCO2_API_ENDPOINT`: API endpoint for IROCO2 service +- `IROCO2_GATEWAY_ENDPOINT`: Gateway endpoint for IROCO2 service +- `IROCO2_API_KEY`: Authentication token for IROCO2 API + +## Migration from CloudFormation + +This module is designed as a direct replacement for the CloudFormation template in `lambda.yaml`. Key improvements include: + +1. **Better Resource Management**: Terraform state tracking +2. **Input Validation**: Variable validation rules +3. **Modular Design**: Reusable across environments +4. **Enhanced Documentation**: Comprehensive usage examples +5. **Flexible Configuration**: More customization options + +## Troubleshooting + +### Common Issues + +1. **KMS Key Access**: Ensure the KMS key policy allows Lambda service access +2. **S3 Permissions**: Verify the Lambda execution role has GetObject permissions +3. **Layer Compatibility**: Ensure the helper scripts layer is compatible with Python 3.11 +4. **Organization ID**: Verify the AWS Organization ID format is correct + +### Debugging + +Check CloudWatch logs for Lambda execution details: + +```bash +aws logs describe-log-groups --log-group-name-prefix "/aws/lambda/IROCO2-CUR-SCRAPPER" +aws logs get-log-events --log-group-name "/aws/lambda/IROCO2-CUR-SCRAPPER" --log-stream-name "LATEST" +``` + +## Development Setup + +### Prerequisites + +Before working with this module, ensure you have the following tools installed: + +- [Terraform](https://www.terraform.io/downloads.html) >= 1.0 +- [pre-commit](https://pre-commit.com/#installation) +- [TFLint](https://github.com/terraform-linters/tflint#installation) +- [tfsec](https://github.com/aquasecurity/tfsec#installation) +- [Checkov](https://www.checkov.io/1.Welcome/Quick%20Start.html) (optional, installed via pip) +- [terraform-docs](https://terraform-docs.io/user-guide/installation/) (optional) + +### Setup Pre-commit Hooks + +This module includes pre-commit configuration that automatically validates your changes: + +```bash +# Navigate to the module directory +cd lambdas/functions/iroco2-client-side-scanner/tf + +# Install pre-commit hooks (one-time setup) +pre-commit install + +# Optional: Run hooks on all files to test setup +pre-commit run --all-files +``` + +### What the Pre-commit Hooks Do + +The pre-commit configuration automatically runs the following checks on every commit: + +1. **General Checks**: + - Remove trailing whitespace + - Fix end-of-file formatting + - Validate YAML and JSON files + - Check for large files and merge conflicts + +2. **Terraform Validation**: + - Format Terraform files (`terraform fmt`) + - Validate Terraform syntax (`terraform validate`) + - Run TFLint for best practices + - Security scanning with tfsec and Checkov + - Update documentation automatically + +3. **Security**: + - Scan for secrets and sensitive data + - Check for security misconfigurations + +### Manual Commands + +If you need to run checks manually: + +```bash +# Format Terraform files +terraform fmt -recursive + +# Validate Terraform configuration +terraform validate + +# Run TFLint +tflint + +# Run security checks +tfsec . +checkov -f . --framework terraform + +# Run all pre-commit hooks manually +pre-commit run --all-files +``` + +## Deployment Guide + +### Prerequisites + +Before deploying this module, ensure you have: + +1. **AWS CLI configured** with appropriate permissions +2. **Terraform installed** (>= 1.0) +3. **Required AWS resources** created (see below) + +### Required AWS Resources + +You need to create these resources before deploying: + +1. **KMS Key** for encryption +2. **S3 Bucket** for Lambda layers with the helper scripts ZIP uploaded +3. **S3 Bucket** for Lambda function code with the function ZIP uploaded +4. **AWS Organization ID** (if using AWS Organizations) +5. **IROCO2 API credentials** and endpoints + +### Step-by-Step Deployment + +1. **Configure AWS Profile**: + ```bash + aws configure --profile default + # or use existing profile + export AWS_PROFILE=default + ``` + +2. **Navigate to module directory**: + ```bash + cd lambdas/functions/iroco2-client-side-scanner/tf + ``` + +3. **Update variables**: + ```bash + # Copy example file + cp terraform.tfvars.example terraform.tfvars + + # Edit with your actual values + nano terraform.tfvars + ``` + +4. **Initialize Terraform**: + ```bash + terraform init + ``` + +5. **Plan deployment**: + ```bash + terraform plan + ``` + +6. **Deploy the module**: + ```bash + terraform apply + ``` + +7. **Verify deployment**: + ```bash + # Check outputs + terraform output + + # Verify Lambda function + aws lambda get-function --function-name IROCO2-CUR-SCRAPPER-dev + ``` + +### Example terraform.tfvars + +```hcl +# Update these with your actual values +kms_key_arn = "arn:aws:kms:eu-west-3:123456789012:key/abcd1234-..." +layer_bucket_storage = "my-lambda-layers-bucket" +layer_bucket_key = "layers/helper-scripts.zip" +aws_org_id = "o-1234567890" +cur_output_bucket_name = "iroco2-cur-output-unique-suffix" +cur_function_s3_key = "functions/cur-scanner.zip" +cur_function_s3_bucket = "my-lambda-functions-bucket" +iroco2_api_endpoint = "https://api.iroco2.example.com" +iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" +iroco2_api_key = "your-secret-api-key" + +common_tags = { + Environment = "production" + Project = "IROCO2" + Owner = "data-team" +} +``` + +### Cleanup + +To destroy the infrastructure: + +```bash +terraform destroy +``` + +**Warning**: This will delete all resources created by this module, including the S3 bucket and any data it contains. + +## Contributing + +When modifying this module: + +1. Ensure pre-commit hooks are installed and passing +2. Update variable descriptions and validation rules +3. Add new outputs for any additional resources +4. Update this README with new features or changes +5. Test the module in a development environment +6. Ensure backward compatibility when possible + +## License + +This module is part of the IROCO2 project and follows the project's licensing terms. diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/cloudwatch.tf b/lambdas/functions/iroco2-client-side-scanner/tf/cloudwatch.tf new file mode 100644 index 0000000..d01565c --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/cloudwatch.tf @@ -0,0 +1,8 @@ +# CloudWatch Log Group for Lambda +resource "aws_cloudwatch_log_group" "lambda" { + name = var.lambda_log_group_name + retention_in_days = var.log_retention_days + kms_key_id = var.kms_key_arn + + tags = local.tags +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/README.md b/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/README.md new file mode 100644 index 0000000..743598d --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/README.md @@ -0,0 +1,121 @@ +# Basic Example - IROCO2 Client Side Scanner + +This example demonstrates how to use the IROCO2 Client Side Scanner Terraform module in your own project. + +## Prerequisites + +1. **AWS Account** with appropriate permissions +2. **Terraform** >= 1.0 installed +3. **AWS CLI** configured with credentials +4. **KMS Key** with ENCRYPT_DECRYPT usage (can use `alias/aws/s3`) +5. **AWS Organization ID** (if using AWS Organizations) +6. **IROCO2 API credentials** and endpoints + +## Quick Start + +1. **Clone or copy this example**: + ```bash + cp -r examples/basic my-iroco2-deployment + cd my-iroco2-deployment + ``` + +2. **Configure variables**: + ```bash + cp terraform.tfvars.example terraform.tfvars + # Edit terraform.tfvars with your actual values + ``` + +3. **Build Lambda artifacts** (from the module root): + ```bash + cd ../../ + zip -r cur-scanner.zip package/ + zip -r helper-scripts.zip layers/ + cd examples/basic/ + ``` + +4. **Deploy**: + ```bash + terraform init + terraform plan + terraform apply + ``` + +## Configuration + +### Required Variables + +| Variable | Description | Example | +|----------|-------------|---------| +| `kms_key_id` | KMS key for encryption | `"alias/aws/s3"` | +| `aws_org_id` | AWS Organization ID | `"o-1234567890"` | +| `iroco2_api_endpoint` | IROCO2 API URL | `"https://api.iroco2.com"` | +| `iroco2_gateway_endpoint` | IROCO2 Gateway URL | `"https://gateway.iroco2.com"` | +| `iroco2_api_key` | IROCO2 API key | `"your-secret-key"` | + +### Optional Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `aws_region` | AWS region | `"eu-west-3"` | +| `project_name` | Project name prefix | `"iroco2"` | +| `environment` | Environment name | `"dev"` | +| `lambda_timeout` | Function timeout (seconds) | `900` | +| `lambda_memory_size` | Function memory (MB) | `512` | + +## What This Example Creates + +- **S3 Bucket**: For Lambda artifacts (automatically named) +- **S3 Bucket**: For CUR output (automatically named) +- **Lambda Function**: CUR processing function +- **Lambda Layer**: Helper scripts layer +- **IAM Role**: With minimal required permissions +- **CloudWatch Log Group**: For monitoring +- **S3 Event Notifications**: Automatic triggering + +## Usage + +After deployment, upload CUR files (`.parquet` or `.csv.zip`) to the CUR output bucket: + +```bash +aws s3 cp my-cur-file.parquet s3://$(terraform output -raw s3_bucket_name)/ +``` + +The Lambda function will automatically process the file and send data to IROCO2. + +## Monitoring + +View logs in CloudWatch: + +```bash +aws logs tail $(terraform output -raw cloudwatch_log_group) --follow +``` + +## Cleanup + +```bash +terraform destroy +``` + +## Customization + +This example can be customized by: + +1. **Using existing S3 buckets** instead of creating new ones +2. **Modifying Lambda configuration** (memory, timeout, environment variables) +3. **Adding additional tags** or resource naming conventions +4. **Integrating with existing VPC** or security groups + +## Troubleshooting + +### Common Issues + +1. **KMS Key Permissions**: Ensure the KMS key allows CloudWatch Logs service +2. **S3 Bucket Names**: Must be globally unique +3. **Lambda Artifacts**: Ensure ZIP files exist before deployment +4. **API Credentials**: Verify IROCO2 endpoints and API key + +### Getting Help + +- Check the main module [README](../../README.md) for detailed documentation +- Review CloudWatch logs for runtime errors +- Validate your terraform.tfvars configuration diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/main.tf b/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/main.tf new file mode 100644 index 0000000..8a8cf4c --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/main.tf @@ -0,0 +1,89 @@ +# Basic example of using the IROCO2 Client Side Scanner module + +terraform { + required_version = ">= 1.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } +} + +provider "aws" { + region = var.aws_region +} + +# Data sources for existing resources +data "aws_kms_key" "existing" { + key_id = var.kms_key_id +} + +# Example S3 buckets (you may want to use existing ones) +resource "aws_s3_bucket" "lambda_artifacts" { + bucket = "${var.project_name}-lambda-artifacts-${random_id.suffix.hex}" +} + +resource "random_id" "suffix" { + byte_length = 4 +} + +# Upload Lambda function and layer (optional - you can use existing S3 objects) +resource "aws_s3_object" "lambda_function" { + bucket = aws_s3_bucket.lambda_artifacts.bucket + key = "functions/cur-scanner.zip" + source = "../../cur-scanner.zip" + etag = filemd5("../../cur-scanner.zip") +} + +resource "aws_s3_object" "lambda_layer" { + bucket = aws_s3_bucket.lambda_artifacts.bucket + key = "layers/helper-scripts.zip" + source = "../../helper-scripts.zip" + etag = filemd5("../../helper-scripts.zip") +} + +# Use the IROCO2 Client Side Scanner module +module "iroco2_scanner" { + source = "../../" + + # Required variables + kms_key_arn = data.aws_kms_key.existing.arn + layer_bucket_storage = aws_s3_bucket.lambda_artifacts.bucket + layer_bucket_key = aws_s3_object.lambda_layer.key + aws_org_id = var.aws_org_id + cur_output_bucket_name = "${var.project_name}-cur-output-${random_id.suffix.hex}" + cur_function_s3_key = aws_s3_object.lambda_function.key + cur_function_s3_bucket = aws_s3_bucket.lambda_artifacts.bucket + iroco2_api_endpoint = var.iroco2_api_endpoint + iroco2_gateway_endpoint = var.iroco2_gateway_endpoint + iroco2_api_key = var.iroco2_api_key + + # Optional customization + lambda_function_name = "${var.project_name}-cur-scanner" + lambda_timeout = var.lambda_timeout + lambda_memory_size = var.lambda_memory_size + + common_tags = { + Environment = var.environment + Project = var.project_name + Owner = var.owner + Repository = "https://github.com/ippontech/iroco2-lambdas" + } +} + +# Outputs +output "lambda_function_arn" { + description = "ARN of the deployed Lambda function" + value = module.iroco2_scanner.lambda_function_arn +} + +output "s3_bucket_name" { + description = "Name of the CUR output S3 bucket" + value = module.iroco2_scanner.s3_bucket_name +} + +output "cloudwatch_log_group" { + description = "CloudWatch log group for monitoring" + value = module.iroco2_scanner.cloudwatch_log_group_name +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/terraform.tfvars.example b/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/terraform.tfvars.example new file mode 100644 index 0000000..4570b92 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/terraform.tfvars.example @@ -0,0 +1,23 @@ +# Example configuration for the IROCO2 Client Side Scanner module +# Copy this file to terraform.tfvars and update with your actual values + +# AWS Configuration +aws_region = "eu-west-3" # Paris region + +# Project Configuration +project_name = "my-iroco2-project" +environment = "dev" +owner = "my-team" + +# Required AWS Resources (you must provide these) +kms_key_id = "alias/aws/s3" # or your KMS key ID +aws_org_id = "o-1234567890" # Your AWS Organization ID + +# IROCO2 API Configuration (update with real endpoints) +iroco2_api_endpoint = "https://api.iroco2.example.com" +iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" +iroco2_api_key = "your-secret-api-key-here" + +# Optional Lambda Configuration +lambda_timeout = 900 # 15 minutes +lambda_memory_size = 512 # MB diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/variables.tf b/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/variables.tf new file mode 100644 index 0000000..a2782a8 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/variables.tf @@ -0,0 +1,67 @@ +# Variables for the basic example + +variable "aws_region" { + description = "AWS region to deploy resources" + type = string + default = "eu-west-3" +} + +variable "project_name" { + description = "Name of the project (used for resource naming)" + type = string + default = "iroco2" +} + +variable "environment" { + description = "Environment name (dev, staging, prod)" + type = string + default = "dev" +} + +variable "owner" { + description = "Owner of the resources" + type = string + default = "data-team" +} + +variable "kms_key_id" { + description = "KMS key ID or alias for encryption (must support ENCRYPT_DECRYPT)" + type = string + # Example: "alias/aws/s3" or "12345678-1234-1234-1234-123456789012" +} + +variable "aws_org_id" { + description = "AWS Organization ID for access control" + type = string + # Example: "o-1234567890" +} + +variable "iroco2_api_endpoint" { + description = "IROCO2 API endpoint URL" + type = string + # Example: "https://api.iroco2.example.com" +} + +variable "iroco2_gateway_endpoint" { + description = "IROCO2 Gateway endpoint URL" + type = string + # Example: "https://gateway.iroco2.example.com" +} + +variable "iroco2_api_key" { + description = "IROCO2 API authentication key" + type = string + sensitive = true +} + +variable "lambda_timeout" { + description = "Lambda function timeout in seconds" + type = number + default = 900 +} + +variable "lambda_memory_size" { + description = "Lambda function memory size in MB" + type = number + default = 512 +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/iam.tf b/lambdas/functions/iroco2-client-side-scanner/tf/iam.tf new file mode 100644 index 0000000..c140222 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/iam.tf @@ -0,0 +1,52 @@ +# IAM role for Lambda execution +resource "aws_iam_role" "lambda_execution" { + name = "${var.lambda_function_name}-execution-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + Action = "sts:AssumeRole" + } + ] + }) + + tags = local.tags +} + +# Attach AWS managed policy for basic Lambda execution +resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { + role = aws_iam_role.lambda_execution.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +# Custom policy for S3 and KMS access +resource "aws_iam_role_policy" "lambda_s3_kms_access" { + name = "${var.lambda_function_name}-s3-kms-policy" + role = aws_iam_role.lambda_execution.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion" + ] + Resource = "${aws_s3_bucket.cur_output.arn}/*" + }, + { + Effect = "Allow" + Action = [ + "kms:Decrypt" + ] + Resource = var.kms_key_arn + } + ] + }) +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/lambda.tf b/lambdas/functions/iroco2-client-side-scanner/tf/lambda.tf new file mode 100644 index 0000000..7214082 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/lambda.tf @@ -0,0 +1,67 @@ +# Lambda Layer for helper scripts +resource "aws_lambda_layer_version" "helper_scripts" { + layer_name = local.layer_name + description = "Scripts for IROCO2 helpers" + compatible_runtimes = ["python3.9", "python3.10", "python3.11"] + + s3_bucket = var.layer_bucket_storage + s3_key = var.layer_bucket_key + + # Layer will be updated when S3 object changes + # source_code_hash can be added if you want to track local changes +} + +# Lambda Function +resource "aws_lambda_function" "processing" { + function_name = var.lambda_function_name + description = "IROCO2 CUR Scanner - Processes CUR data for carbon impact analysis" + + role = aws_iam_role.lambda_execution.arn + handler = "cur_scrapper.handler" + runtime = "python3.11" + + timeout = var.lambda_timeout + memory_size = var.lambda_memory_size + + # Lambda layers + layers = [ + aws_lambda_layer_version.helper_scripts.arn, + var.aws_sdk_pandas_layer_arn + ] + + # Source code from S3 + s3_bucket = var.cur_function_s3_bucket + s3_key = var.cur_function_s3_key + + # Environment variables + environment { + variables = { + IROCO2_API_ENDPOINT = var.iroco2_api_endpoint + IROCO2_GATEWAY_ENDPOINT = var.iroco2_gateway_endpoint + IROCO2_API_KEY = var.iroco2_api_key + } + } + + # Logging configuration + logging_config { + log_format = "Text" + log_group = aws_cloudwatch_log_group.lambda.name + } + + tags = local.tags + + depends_on = [ + aws_iam_role_policy_attachment.lambda_basic_execution, + aws_iam_role_policy.lambda_s3_kms_access, + aws_cloudwatch_log_group.lambda + ] +} + +# Lambda permission for S3 to invoke the function +resource "aws_lambda_permission" "s3_invoke" { + statement_id = "AllowExecutionFromS3Bucket" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.processing.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.cur_output.arn +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/locals.tf b/lambdas/functions/iroco2-client-side-scanner/tf/locals.tf new file mode 100644 index 0000000..6b07769 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/locals.tf @@ -0,0 +1,30 @@ +locals { + # Common tags merged with user-provided tags + tags = merge(var.common_tags, { + ManagedBy = "Terraform" + Module = "iroco2-client-side-scanner" + }) + + # Lambda layer name + layer_name = "helper-scripts" + + # S3 bucket notification events + s3_notification_events = [ + "s3:ObjectCreated:Copy" + ] + + # S3 object filters for Lambda triggers + s3_object_filters = [ + { + suffix = ".parquet" + }, + { + suffix = ".csv.zip" + } + ] +} + +# Data sources +data "aws_caller_identity" "current" {} + +data "aws_region" "current" {} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/main.tf b/lambdas/functions/iroco2-client-side-scanner/tf/main.tf new file mode 100644 index 0000000..e84dfc1 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/main.tf @@ -0,0 +1,5 @@ +# Main Terraform configuration for IROCO2 Client Side Scanner +# This module creates all necessary AWS resources for the CUR scanner Lambda function + +# This file serves as the main entry point and can include any additional +# configuration or data sources that don't fit in the specific resource files diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/outputs.tf b/lambdas/functions/iroco2-client-side-scanner/tf/outputs.tf new file mode 100644 index 0000000..2a5325c --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/outputs.tf @@ -0,0 +1,59 @@ +output "lambda_function_name" { + description = "Name of the created Lambda function for CUR extraction" + value = aws_lambda_function.processing.function_name +} + +output "lambda_function_arn" { + description = "ARN of the created Lambda function" + value = aws_lambda_function.processing.arn +} + +output "lambda_function_invoke_arn" { + description = "Invoke ARN of the Lambda function" + value = aws_lambda_function.processing.invoke_arn +} + +output "s3_bucket_name" { + description = "Name of the S3 bucket for CUR output" + value = aws_s3_bucket.cur_output.bucket +} + +output "s3_bucket_arn" { + description = "ARN of the S3 bucket for CUR output" + value = aws_s3_bucket.cur_output.arn +} + +output "s3_bucket_domain_name" { + description = "Domain name of the S3 bucket" + value = aws_s3_bucket.cur_output.bucket_domain_name +} + +output "lambda_execution_role_arn" { + description = "ARN of the Lambda execution role" + value = aws_iam_role.lambda_execution.arn +} + +output "lambda_execution_role_name" { + description = "Name of the Lambda execution role" + value = aws_iam_role.lambda_execution.name +} + +output "lambda_layer_arn" { + description = "ARN of the helper scripts Lambda layer" + value = aws_lambda_layer_version.helper_scripts.arn +} + +output "lambda_layer_version" { + description = "Version of the helper scripts Lambda layer" + value = aws_lambda_layer_version.helper_scripts.version +} + +output "cloudwatch_log_group_name" { + description = "Name of the CloudWatch log group" + value = aws_cloudwatch_log_group.lambda.name +} + +output "cloudwatch_log_group_arn" { + description = "ARN of the CloudWatch log group" + value = aws_cloudwatch_log_group.lambda.arn +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/providers.tf b/lambdas/functions/iroco2-client-side-scanner/tf/providers.tf new file mode 100644 index 0000000..b7861b3 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/providers.tf @@ -0,0 +1,11 @@ +provider "aws" { + region = "eu-west-3" + profile = "default" + + default_tags { + tags = { + ManagedBy = "Terraform" + Module = "iroco2-client-side-scanner" + } + } +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response.json b/lambdas/functions/iroco2-client-side-scanner/tf/response.json new file mode 100644 index 0000000..a92b7ba --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/response.json @@ -0,0 +1 @@ +{"errorMessage": "Unable to import module 'cur_scrapper': No module named 'cur_scrapper'", "errorType": "Runtime.ImportModuleError", "requestId": "", "stackTrace": []} \ No newline at end of file diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response2.json b/lambdas/functions/iroco2-client-side-scanner/tf/response2.json new file mode 100644 index 0000000..2dd985e --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/response2.json @@ -0,0 +1 @@ +{"errorMessage": "Unable to import module 'cur_scrapper': No module named 's3_repository_parser'", "errorType": "Runtime.ImportModuleError", "requestId": "", "stackTrace": []} \ No newline at end of file diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response3.json b/lambdas/functions/iroco2-client-side-scanner/tf/response3.json new file mode 100644 index 0000000..36f18ec --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/response3.json @@ -0,0 +1 @@ +{"errorMessage": "'Records'", "errorType": "KeyError", "requestId": "26fa1e0b-59b8-4faa-9bec-af6126e7244c", "stackTrace": [" File \"/var/task/cur_scrapper.py\", line 54, in handler\n print(f\"Event Name: {event['Records'][0]['eventName']}\")\n"]} \ No newline at end of file diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response4.json b/lambdas/functions/iroco2-client-side-scanner/tf/response4.json new file mode 100644 index 0000000..aa1205b --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/response4.json @@ -0,0 +1 @@ +{"errorMessage": "HTTPSConnectionPool(host='api.iroco2.example.com', port=443): Max retries exceeded with url: / (Caused by NewConnectionError(': Failed to establish a new connection: [Errno -2] Name or service not known'))", "errorType": "ConnectionError", "requestId": "f52f8f03-07bb-431d-8451-496f74e1f4d2", "stackTrace": [" File \"/var/task/cur_scrapper.py\", line 59, in handler\n cur_processor_service.parsing_cur_and_send_to_api(\n", " File \"/var/task/cur_scrapper.py\", line 42, in parsing_cur_and_send_to_api\n self.__send_message_parsed_to_api(message_parsed)\n", " File \"/var/task/cur_scrapper.py\", line 46, in __send_message_parsed_to_api\n correlation_id = self.iroco_service.create_scan()\n", " File \"/opt/python/iroco_service.py\", line 40, in create_scan\n response = requests.post(self.iroco_endpoint, headers=self.__build_header())\n", " File \"/opt/python/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n", " File \"/opt/python/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 589, in request\n resp = self.send(prep, **send_kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 703, in send\n r = adapter.send(request, **kwargs)\n", " File \"/opt/python/requests/adapters.py\", line 700, in send\n raise ConnectionError(e, request=request)\n"]} \ No newline at end of file diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response5.json b/lambdas/functions/iroco2-client-side-scanner/tf/response5.json new file mode 100644 index 0000000..01f7003 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/response5.json @@ -0,0 +1 @@ +{"errorMessage": "HTTPSConnectionPool(host='api.iroco2.example.com', port=443): Max retries exceeded with url: / (Caused by NewConnectionError(': Failed to establish a new connection: [Errno -2] Name or service not known'))", "errorType": "ConnectionError", "requestId": "5fd110a3-d450-463d-ac4b-2bdf0e5931a6", "stackTrace": [" File \"/var/task/cur_scrapper.py\", line 60, in handler\n cur_processor_service.parsing_cur_and_send_to_api(\n", " File \"/var/task/cur_scrapper.py\", line 42, in parsing_cur_and_send_to_api\n self.__send_message_parsed_to_api(message_parsed)\n", " File \"/var/task/cur_scrapper.py\", line 46, in __send_message_parsed_to_api\n correlation_id = self.iroco_service.create_scan()\n", " File \"/opt/python/iroco_service.py\", line 40, in create_scan\n response = requests.post(self.iroco_endpoint, headers=self.__build_header())\n", " File \"/opt/python/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n", " File \"/opt/python/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 589, in request\n resp = self.send(prep, **send_kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 703, in send\n r = adapter.send(request, **kwargs)\n", " File \"/opt/python/requests/adapters.py\", line 700, in send\n raise ConnectionError(e, request=request)\n"]} \ No newline at end of file diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/s3.tf b/lambdas/functions/iroco2-client-side-scanner/tf/s3.tf new file mode 100644 index 0000000..c3d79c9 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/s3.tf @@ -0,0 +1,97 @@ +# S3 bucket for CUR output +resource "aws_s3_bucket" "cur_output" { + bucket = var.cur_output_bucket_name + + tags = local.tags +} + +# S3 bucket versioning +resource "aws_s3_bucket_versioning" "cur_output" { + bucket = aws_s3_bucket.cur_output.id + versioning_configuration { + status = "Suspended" + } +} + +# S3 bucket encryption +resource "aws_s3_bucket_server_side_encryption_configuration" "cur_output" { + bucket = aws_s3_bucket.cur_output.id + + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = var.kms_key_arn + sse_algorithm = "aws:kms" + } + } +} + +# S3 bucket public access block +resource "aws_s3_bucket_public_access_block" "cur_output" { + bucket = aws_s3_bucket.cur_output.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +# S3 bucket policy +resource "aws_s3_bucket_policy" "cur_output" { + bucket = aws_s3_bucket.cur_output.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + AWS = data.aws_caller_identity.current.account_id + } + Action = "s3:*" + Resource = [ + aws_s3_bucket.cur_output.arn, + "${aws_s3_bucket.cur_output.arn}/*" + ] + Condition = { + Bool = { + "aws:SecureTransport" = "true" + } + StringEquals = { + "aws:PrincipalOrgId" = var.aws_org_id + } + } + }, + { + Effect = "Deny" + Principal = { + AWS = aws_iam_role.lambda_execution.arn + } + NotAction = "s3:GetObject" + Resource = "${aws_s3_bucket.cur_output.arn}/*" + Condition = { + Bool = { + "aws:SecureTransport" = "true" + } + } + } + ] + }) + + depends_on = [aws_s3_bucket_public_access_block.cur_output] +} + +# S3 bucket notification +resource "aws_s3_bucket_notification" "cur_output" { + bucket = aws_s3_bucket.cur_output.id + + dynamic "lambda_function" { + for_each = local.s3_object_filters + content { + lambda_function_arn = aws_lambda_function.processing.arn + events = local.s3_notification_events + filter_suffix = lambda_function.value.suffix + } + } + + depends_on = [aws_lambda_permission.s3_invoke] +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/terraform.tfvars.example b/lambdas/functions/iroco2-client-side-scanner/tf/terraform.tfvars.example new file mode 100644 index 0000000..d46fcc2 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/terraform.tfvars.example @@ -0,0 +1,29 @@ +# Example Terraform variables for IROCO2 Client Side Scanner +# Copy this file to terraform.tfvars and update with your actual values + +# Required variables +kms_key_arn = "arn:aws:kms:eu-west-3:992382519845:key/02094aa9-afab-4e3c-84b2-8082380c66eb" +layer_bucket_storage = "iroco-test-sylvain-scrapper" +layer_bucket_key = "layers/helper-scripts.zip" +aws_org_id = "o-yn7toj4ob7" +cur_output_bucket_name = "iroco-cur-bucket" +cur_function_s3_key = "functions/cur-scanner.zip" +cur_function_s3_bucket = "iroco-test-sylvain-scrapper" +iroco2_api_endpoint = "https://api.iroco2.example.com" +iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" +iroco2_api_key = "your-api-key-here" + +# Optional variables (uncomment to override defaults) +# lambda_function_name = "IROCO2-CUR-SCRAPPER" +# lambda_log_group_name = "/aws/lambda/IROCO2-CUR-SCRAPPER" +# lambda_timeout = 900 +# lambda_memory_size = 512 +# log_retention_days = 14 + +# Common tags +common_tags = { + Environment = "development" + Project = "IROCO2" + Owner = "data-team" + ManagedBy = "Terraform" +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/test-payload.json b/lambdas/functions/iroco2-client-side-scanner/tf/test-payload.json new file mode 100644 index 0000000..1c04e1b --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/test-payload.json @@ -0,0 +1,15 @@ +{ + "Records": [ + { + "eventName": "ObjectCreated:Copy", + "s3": { + "bucket": { + "name": "iroco-cur-bucket" + }, + "object": { + "key": "iroco/iroco_CUR_report/20250901-20251001/20250918T091302Z/iroco_CUR_report-00001.csv.zip" + } + } + } + ] +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/variables.tf b/lambdas/functions/iroco2-client-side-scanner/tf/variables.tf new file mode 100644 index 0000000..6afa01a --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/variables.tf @@ -0,0 +1,128 @@ +variable "kms_key_arn" { + description = "The KMS key ARN to encrypt the bucket and logs" + type = string + validation { + condition = can(regex("^arn:aws:kms:", var.kms_key_arn)) + error_message = "The KMS key ARN must be a valid AWS KMS key ARN." + } +} + +variable "layer_bucket_storage" { + description = "The S3 bucket name for the lambda layer" + type = string +} + +variable "layer_bucket_key" { + description = "The S3 bucket key for the lambda layer" + type = string +} + +variable "aws_org_id" { + description = "The AWS Organization ID" + type = string + validation { + condition = can(regex("^o-[a-z0-9]{10,32}$", var.aws_org_id)) + error_message = "The AWS Organization ID must be in the format o-xxxxxxxxxx." + } +} + +variable "cur_output_bucket_name" { + description = "The S3 bucket name for the CUR output" + type = string + validation { + condition = can(regex("^[a-z0-9][a-z0-9-]*[a-z0-9]$", var.cur_output_bucket_name)) + error_message = "The bucket name must be a valid S3 bucket name." + } +} + +variable "cur_function_s3_key" { + description = "The S3 bucket key for the CUR lambda function" + type = string +} + +variable "cur_function_s3_bucket" { + description = "The S3 bucket name for the CUR lambda function" + type = string +} + +variable "iroco2_api_endpoint" { + description = "The IroCO2 API endpoint" + type = string + validation { + condition = can(regex("^https?://", var.iroco2_api_endpoint)) + error_message = "The API endpoint must be a valid HTTP or HTTPS URL." + } +} + +variable "iroco2_gateway_endpoint" { + description = "The IroCO2 Gateway endpoint" + type = string + validation { + condition = can(regex("^https?://", var.iroco2_gateway_endpoint)) + error_message = "The Gateway endpoint must be a valid HTTP or HTTPS URL." + } +} + +variable "iroco2_api_key" { + description = "The IroCO2 API token" + type = string + sensitive = true +} + +variable "lambda_log_group_name" { + description = "Log group name for lambda" + type = string + default = "/aws/lambda/IROCO2-CUR-SCRAPPER" +} + +variable "lambda_function_name" { + description = "Name of the Lambda function" + type = string + default = "IROCO2-CUR-SCRAPPER" +} + +variable "lambda_timeout" { + description = "Lambda function timeout in seconds" + type = number + default = 900 + validation { + condition = var.lambda_timeout >= 1 && var.lambda_timeout <= 900 + error_message = "Lambda timeout must be between 1 and 900 seconds." + } +} + +variable "lambda_memory_size" { + description = "Lambda function memory size in MB" + type = number + default = 512 + validation { + condition = var.lambda_memory_size >= 128 && var.lambda_memory_size <= 10240 + error_message = "Lambda memory size must be between 128 and 10240 MB." + } +} + +variable "log_retention_days" { + description = "CloudWatch log retention in days" + type = number + default = 14 + validation { + condition = contains([ + 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653 + ], var.log_retention_days) + error_message = "Log retention days must be a valid CloudWatch Logs retention value." + } +} + +variable "common_tags" { + description = "Common tags to apply to all resources" + type = map(string) + default = { + Project = "IROCO2" + } +} + +variable "aws_sdk_pandas_layer_arn" { + description = "ARN of the AWS SDK Pandas layer" + type = string + default = "arn:aws:lambda:eu-west-3:336392948345:layer:AWSSDKPandas-Python311:18" +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/versions.tf b/lambdas/functions/iroco2-client-side-scanner/tf/versions.tf new file mode 100644 index 0000000..ddfcb0e --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/tf/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } +} From 5a8bee808c1f61f352ca8c1ef63fcffdda7ea754 Mon Sep 17 00:00:00 2001 From: sressier Date: Thu, 18 Sep 2025 14:40:03 +0200 Subject: [PATCH 2/9] update gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d24eab2..545f95f 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,4 @@ venv/ *tfvars *override*.tf .terraform -.terraform.lock.hcl \ No newline at end of file +.terraform.lock.hcl From 62c221674d160f823b31dee81105b50eea7944e0 Mon Sep 17 00:00:00 2001 From: sressier Date: Fri, 26 Sep 2025 09:54:41 +0200 Subject: [PATCH 3/9] move files --- .../iroco2-client-side-scanner/.gitignore | 22 + .../.pre-commit-config.yaml | 43 ++ .../iroco2-client-side-scanner/.tflint.hcl | 95 ++++ .../iroco2-client-side-scanner/README.md | 453 +++++++++++++++++- .../iroco2-client-side-scanner/cloudwatch.tf | 8 + .../cur_scrapper.py | 62 --- .../{tf => }/examples/basic/README.md | 0 .../{tf => }/examples/basic/main.tf | 17 +- .../examples/basic/terraform.tfvars.example | 0 .../{tf => }/examples/basic/variables.tf | 0 .../iroco2-client-side-scanner/iam.tf | 52 ++ .../iroco2-client-side-scanner/lambda.tf | 70 +++ .../iroco2-client-side-scanner/lambda.yaml | 193 -------- .../iroco2-client-side-scanner/locals.tf | 30 ++ .../iroco2-client-side-scanner/main.tf | 5 + .../iroco2-client-side-scanner/outputs.tf | 59 +++ .../iroco2-client-side-scanner/providers.tf | 11 + .../iroco2-client-side-scanner/s3.tf | 97 ++++ .../terraform.tfvars.example | 29 ++ .../test-payload.json | 15 + .../iroco2-client-side-scanner/tf/README.md | 4 +- .../tf/response.json | 2 +- .../tf/response2.json | 2 +- .../tf/response3.json | 2 +- .../tf/response4.json | 2 +- .../tf/response5.json | 2 +- .../iroco2-client-side-scanner/variables.tf | 128 +++++ .../iroco2-client-side-scanner/versions.tf | 10 + 28 files changed, 1129 insertions(+), 284 deletions(-) create mode 100644 lambdas/functions/iroco2-client-side-scanner/.gitignore create mode 100644 lambdas/functions/iroco2-client-side-scanner/.pre-commit-config.yaml create mode 100644 lambdas/functions/iroco2-client-side-scanner/.tflint.hcl create mode 100644 lambdas/functions/iroco2-client-side-scanner/cloudwatch.tf delete mode 100644 lambdas/functions/iroco2-client-side-scanner/cur_scrapper.py rename lambdas/functions/iroco2-client-side-scanner/{tf => }/examples/basic/README.md (100%) rename lambdas/functions/iroco2-client-side-scanner/{tf => }/examples/basic/main.tf (79%) rename lambdas/functions/iroco2-client-side-scanner/{tf => }/examples/basic/terraform.tfvars.example (100%) rename lambdas/functions/iroco2-client-side-scanner/{tf => }/examples/basic/variables.tf (100%) create mode 100644 lambdas/functions/iroco2-client-side-scanner/iam.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/lambda.tf delete mode 100644 lambdas/functions/iroco2-client-side-scanner/lambda.yaml create mode 100644 lambdas/functions/iroco2-client-side-scanner/locals.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/main.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/outputs.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/providers.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/s3.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/terraform.tfvars.example create mode 100644 lambdas/functions/iroco2-client-side-scanner/test-payload.json create mode 100644 lambdas/functions/iroco2-client-side-scanner/variables.tf create mode 100644 lambdas/functions/iroco2-client-side-scanner/versions.tf diff --git a/lambdas/functions/iroco2-client-side-scanner/.gitignore b/lambdas/functions/iroco2-client-side-scanner/.gitignore new file mode 100644 index 0000000..fe5ac31 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/.gitignore @@ -0,0 +1,22 @@ +# Terraform files +*.tfstate +*.tfstate.* +*.tfvars +!terraform.tfvars.example +.terraform/ +.terraform.lock.hcl +crash.log +crash.*.log + +# IDE files +.vscode/ +.idea/ +*.swp +*.swo + +# OS files +.DS_Store +Thumbs.db + +# Pre-commit +.secrets.baseline diff --git a/lambdas/functions/iroco2-client-side-scanner/.pre-commit-config.yaml b/lambdas/functions/iroco2-client-side-scanner/.pre-commit-config.yaml new file mode 100644 index 0000000..fda3340 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/.pre-commit-config.yaml @@ -0,0 +1,43 @@ +# Simplified pre-commit configuration for IROCO2 Client Side Scanner Terraform module +# This configuration includes only essential checks that work without additional tool installation +repos: + # General hooks + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + + # Terraform hooks (minimal set) + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.100.0 + hooks: + - id: terraform_fmt + description: Rewrites all Terraform configuration files to a canonical format + exclude : lambdas/functions/iroco2-client-side-scanner/examples/ + files : ^lambdas/functions/iroco2-client-side-scanner/ + + - id: terraform_validate + description: Validates all Terraform configuration files + exclude : lambdas/functions/iroco2-client-side-scanner/examples/ + files : ^lambdas/functions/iroco2-client-side-scanner/ + args: + - --hook-config=--retry-once-with-cleanup=true + +# Configuration for specific hooks +default_language_version: + python: python3 + +# Global excludes +exclude: | + (?x)^( + \.terraform/.*| + \.terraform\.lock\.hcl| + terraform\.tfstate.*| + \.git/.*| + examples\.* + )$ diff --git a/lambdas/functions/iroco2-client-side-scanner/.tflint.hcl b/lambdas/functions/iroco2-client-side-scanner/.tflint.hcl new file mode 100644 index 0000000..924d8ff --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/.tflint.hcl @@ -0,0 +1,95 @@ +# TFLint configuration for IROCO2 Client Side Scanner Terraform module + +config { + # Enable all rules by default + disabled_by_default = false + + # Plugin directory + plugin_dir = "~/.tflint.d/plugins" + + # Call module inspection + call_module_type = "all" +} + +# AWS plugin configuration +plugin "aws" { + enabled = true + version = "0.29.0" + source = "github.com/terraform-linters/tflint-ruleset-aws" + + # Deep checking requires AWS credentials + deep_check = false +} + +# Terraform plugin configuration +plugin "terraform" { + enabled = true + preset = "recommended" +} + +# Rule configurations +rule "terraform_deprecated_interpolation" { + enabled = true +} + +rule "terraform_deprecated_index" { + enabled = true +} + +rule "terraform_unused_declarations" { + enabled = true +} + +rule "terraform_comment_syntax" { + enabled = true +} + +rule "terraform_documented_outputs" { + enabled = true +} + +rule "terraform_documented_variables" { + enabled = true +} + +rule "terraform_typed_variables" { + enabled = true +} + +rule "terraform_module_pinned_source" { + enabled = true +} + +rule "terraform_naming_convention" { + enabled = true + format = "snake_case" +} + +rule "terraform_required_version" { + enabled = true +} + +rule "terraform_required_providers" { + enabled = true +} + +rule "terraform_standard_module_structure" { + enabled = true +} + +# AWS specific rules +rule "aws_instance_invalid_type" { + enabled = true +} + +rule "aws_lambda_function_invalid_runtime" { + enabled = true +} + +rule "aws_s3_bucket_invalid_region" { + enabled = true +} + +rule "aws_iam_policy_invalid_policy" { + enabled = true +} diff --git a/lambdas/functions/iroco2-client-side-scanner/README.md b/lambdas/functions/iroco2-client-side-scanner/README.md index 8256aa0..0f00a2e 100644 --- a/lambdas/functions/iroco2-client-side-scanner/README.md +++ b/lambdas/functions/iroco2-client-side-scanner/README.md @@ -1,11 +1,452 @@ -# Iroco2 Client Side Scanner +# IROCO2 Client Side Scanner Terraform Module -## Context +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Terraform](https://img.shields.io/badge/Terraform-%3E%3D1.0-623CE4)](https://www.terraform.io/) +[![AWS](https://img.shields.io/badge/AWS-%3E%3D5.0-FF9900)](https://aws.amazon.com/) -This lambda is used to scan aws accounts to retrieve the CUR data and carbon impact data. +This Terraform module creates the AWS infrastructure for the IROCO2 Client Side Scanner, which processes Cost and Usage Report (CUR) data for carbon impact analysis. -## TODO +> **Open Source Ready**: This module is designed for easy integration into open source projects and can be used by anyone to analyze AWS carbon footprint data. -- Upate docs to self host assets +## Quick Start -SaaS mode documentation (french): +Want to get started quickly? Check out our [basic example](examples/basic/) which includes: + +- πŸ“‹ **Step-by-step setup guide** +- πŸ”§ **Pre-configured variables** +- πŸ“– **Detailed documentation** +- πŸš€ **Ready-to-deploy configuration** + +```bash +# Clone the example +cp -r examples/basic my-iroco2-deployment +cd my-iroco2-deployment + +# Configure your settings +cp terraform.tfvars.example terraform.tfvars +# Edit terraform.tfvars with your AWS details + +# Deploy +terraform init && terraform apply +``` + +## Overview + +The module provisions the following AWS resources: + +- **S3 Bucket**: Encrypted storage for CUR output with event notifications +- **Lambda Function**: Python 3.11 runtime for processing CUR data +- **Lambda Layer**: Custom helper scripts for IROCO2 functionality +- **IAM Role & Policies**: Execution permissions for Lambda with least privilege access +- **CloudWatch Log Group**: Centralized logging with KMS encryption +- **S3 Bucket Policy**: Organization-based access control +- **Lambda Permissions**: S3 trigger configuration + +## Architecture + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ S3 Bucket │───▢│ Lambda Function │───▢│ IROCO2 API β”‚ +β”‚ (CUR Output) β”‚ β”‚ (CUR Scanner) β”‚ β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β”‚ β–Ό + β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ β”‚ CloudWatch Logs β”‚ + β”‚ β”‚ β”‚ + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ KMS Key β”‚ +β”‚ (Encryption) β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +## Usage + +### Basic Usage + +```hcl +module "iroco2_client_side_scanner" { + source = "./lambdas/functions/iroco2-client-side-scanner/tf" + + # Required variables + kms_key_arn = "arn:aws:kms:eu-west-3:123456789012:key/12345678-1234-1234-1234-123456789012" + layer_bucket_storage = "my-lambda-layers-bucket" + layer_bucket_key = "layers/helper-scripts.zip" + aws_org_id = "o-1234567890" + cur_output_bucket_name = "my-cur-output-bucket" + cur_function_s3_key = "functions/cur-scanner.zip" + cur_function_s3_bucket = "my-lambda-functions-bucket" + iroco2_api_endpoint = "https://api.iroco2.example.com" + iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" + iroco2_api_key = "your-api-key-here" + + # Optional variables with defaults + lambda_function_name = "IROCO2-CUR-SCRAPPER" + lambda_log_group_name = "/aws/lambda/IROCO2-CUR-SCRAPPER" + lambda_timeout = 900 + lambda_memory_size = 512 + log_retention_days = 14 + + common_tags = { + Environment = "production" + Project = "IROCO2" + Owner = "data-team" + } +} +``` + +### Advanced Usage with Custom Configuration + +```hcl +module "iroco2_client_side_scanner" { + source = "./lambdas/functions/iroco2-client-side-scanner/tf" + + # Required variables + kms_key_arn = data.aws_kms_key.main.arn + layer_bucket_storage = aws_s3_bucket.lambda_layers.bucket + layer_bucket_key = "layers/helper-scripts-v2.zip" + aws_org_id = var.organization_id + cur_output_bucket_name = "${var.environment}-cur-output-${random_id.bucket_suffix.hex}" + cur_function_s3_key = "functions/cur-scanner-${var.version}.zip" + cur_function_s3_bucket = aws_s3_bucket.lambda_functions.bucket + iroco2_api_endpoint = var.iroco2_api_endpoint + iroco2_gateway_endpoint = var.iroco2_gateway_endpoint + iroco2_api_key = data.aws_secretsmanager_secret_version.iroco2_api_key.secret_string + + # Custom configuration + lambda_function_name = "${var.environment}-iroco2-cur-scanner" + lambda_log_group_name = "/aws/lambda/${var.environment}-iroco2-cur-scanner" + lambda_timeout = 600 + lambda_memory_size = 1024 + log_retention_days = 30 + + common_tags = merge(var.common_tags, { + Environment = var.environment + Module = "iroco2-client-side-scanner" + }) +} +``` + +## Requirements + +| Name | Version | +|------|---------| +| terraform | >= 1.0 | +| aws | >= 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| aws | >= 5.0 | + +## Resources + +| Name | Type | +|------|------| +| aws_s3_bucket.cur_output | resource | +| aws_s3_bucket_versioning.cur_output | resource | +| aws_s3_bucket_server_side_encryption_configuration.cur_output | resource | +| aws_s3_bucket_public_access_block.cur_output | resource | +| aws_s3_bucket_policy.cur_output | resource | +| aws_s3_bucket_notification.cur_output | resource | +| aws_lambda_function.processing | resource | +| aws_lambda_layer_version.helper_scripts | resource | +| aws_lambda_permission.s3_invoke | resource | +| aws_iam_role.lambda_execution | resource | +| aws_iam_role_policy_attachment.lambda_basic_execution | resource | +| aws_iam_role_policy.lambda_s3_kms_access | resource | +| aws_cloudwatch_log_group.lambda | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| kms_key_arn | The KMS key ARN to encrypt the bucket and logs | `string` | n/a | yes | +| layer_bucket_storage | The S3 bucket name for the lambda layer | `string` | n/a | yes | +| layer_bucket_key | The S3 bucket key for the lambda layer | `string` | n/a | yes | +| aws_org_id | The AWS Organization ID | `string` | n/a | yes | +| cur_output_bucket_name | The S3 bucket name for the CUR output | `string` | n/a | yes | +| cur_function_s3_key | The S3 bucket key for the CUR lambda function | `string` | n/a | yes | +| cur_function_s3_bucket | The S3 bucket name for the CUR lambda function | `string` | n/a | yes | +| iroco2_api_endpoint | The IroCO2 API endpoint | `string` | n/a | yes | +| iroco2_gateway_endpoint | The IroCO2 Gateway endpoint | `string` | n/a | yes | +| iroco2_api_key | The IroCO2 API token | `string` | n/a | yes | +| lambda_function_name | Name of the Lambda function | `string` | `"IROCO2-CUR-SCRAPPER"` | no | +| lambda_log_group_name | Log group name for lambda | `string` | `"/aws/lambda/IROCO2-CUR-SCRAPPER"` | no | +| lambda_timeout | Lambda function timeout in seconds | `number` | `900` | no | +| lambda_memory_size | Lambda function memory size in MB | `number` | `512` | no | +| log_retention_days | CloudWatch log retention in days | `number` | `14` | no | +| common_tags | Common tags to apply to all resources | `map(string)` | `{"Project": "IROCO2"}` | no | +| aws_sdk_pandas_layer_arn | ARN of the AWS SDK Pandas layer | `string` | `"arn:aws:lambda:eu-west-3:336392948345:layer:AWSSDKPandas-Python311:18"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| lambda_function_name | Name of the created Lambda function for CUR extraction | +| lambda_function_arn | ARN of the created Lambda function | +| lambda_function_invoke_arn | Invoke ARN of the Lambda function | +| s3_bucket_name | Name of the S3 bucket for CUR output | +| s3_bucket_arn | ARN of the S3 bucket for CUR output | +| s3_bucket_domain_name | Domain name of the S3 bucket | +| lambda_execution_role_arn | ARN of the Lambda execution role | +| lambda_execution_role_name | Name of the Lambda execution role | +| lambda_layer_arn | ARN of the helper scripts Lambda layer | +| lambda_layer_version | Version of the helper scripts Lambda layer | +| cloudwatch_log_group_name | Name of the CloudWatch log group | +| cloudwatch_log_group_arn | ARN of the CloudWatch log group | + +## Features + +### Security + +- **KMS Encryption**: All data at rest is encrypted using customer-managed KMS keys +- **IAM Least Privilege**: Lambda execution role has minimal required permissions +- **Organization Access Control**: S3 bucket policy restricts access to organization members +- **Secure Transport**: All S3 operations require HTTPS + +### Monitoring & Logging + +- **CloudWatch Integration**: Centralized logging with configurable retention +- **Structured Logging**: Lambda function uses structured logging format +- **Error Tracking**: Failed executions are logged for troubleshooting + +### Scalability + +- **Configurable Resources**: Memory and timeout can be adjusted based on workload +- **Layer Management**: Helper scripts are packaged as reusable Lambda layers +- **Event-Driven**: S3 events automatically trigger processing + +## Triggers + +The Lambda function is automatically triggered by S3 events: + +- **Event Type**: `s3:ObjectCreated:Copy` +- **File Filters**: + - Files ending with `.parquet` + - Files ending with `.csv.zip` + +## Environment Variables + +The Lambda function receives the following environment variables: + +- `IROCO2_API_ENDPOINT`: API endpoint for IROCO2 service +- `IROCO2_GATEWAY_ENDPOINT`: Gateway endpoint for IROCO2 service +- `IROCO2_API_KEY`: Authentication token for IROCO2 API + +## Migration from CloudFormation + +This module is designed as a direct replacement for the CloudFormation template in `lambda.yaml`. Key improvements include: + +1. **Better Resource Management**: Terraform state tracking +2. **Input Validation**: Variable validation rules +3. **Modular Design**: Reusable across environments +4. **Enhanced Documentation**: Comprehensive usage examples +5. **Flexible Configuration**: More customization options + +## Troubleshooting + +### Common Issues + +1. **KMS Key Access**: Ensure the KMS key policy allows Lambda service access +2. **S3 Permissions**: Verify the Lambda execution role has GetObject permissions +3. **Layer Compatibility**: Ensure the helper scripts layer is compatible with Python 3.11 +4. **Organization ID**: Verify the AWS Organization ID format is correct + +### Debugging + +Check CloudWatch logs for Lambda execution details: + +```bash +aws logs describe-log-groups --log-group-name-prefix "/aws/lambda/IROCO2-CUR-SCRAPPER" +aws logs get-log-events --log-group-name "/aws/lambda/IROCO2-CUR-SCRAPPER" --log-stream-name "LATEST" +``` + +## Development Setup + +### Prerequisites + +Before working with this module, ensure you have the following tools installed: + +- [Terraform](https://www.terraform.io/downloads.html) >= 1.0 +- [pre-commit](https://pre-commit.com/#installation) +- [TFLint](https://github.com/terraform-linters/tflint#installation) +- [tfsec](https://github.com/aquasecurity/tfsec#installation) +- [Checkov](https://www.checkov.io/1.Welcome/Quick%20Start.html) (optional, installed via pip) +- [terraform-docs](https://terraform-docs.io/user-guide/installation/) (optional) + +### Setup Pre-commit Hooks + +This module includes pre-commit configuration that automatically validates your changes: + +```bash +# Navigate to the module directory +cd lambdas/functions/iroco2-client-side-scanner/tf + +# Install pre-commit hooks (one-time setup) +pre-commit install + +# Optional: Run hooks on all files to test setup +pre-commit run --all-files +``` + +### What the Pre-commit Hooks Do + +The pre-commit configuration automatically runs the following checks on every commit: + +1. **General Checks**: + - Remove trailing whitespace + - Fix end-of-file formatting + - Validate YAML and JSON files + - Check for large files and merge conflicts + +2. **Terraform Validation**: + - Format Terraform files (`terraform fmt`) + - Validate Terraform syntax (`terraform validate`) + - Run TFLint for best practices + - Security scanning with tfsec and Checkov + - Update documentation automatically + +3. **Security**: + - Scan for secrets and sensitive data + - Check for security misconfigurations + +### Manual Commands + +If you need to run checks manually: + +```bash +# Format Terraform files +terraform fmt -recursive + +# Validate Terraform configuration +terraform validate + +# Run TFLint +tflint + +# Run security checks +tfsec . +checkov -f . --framework terraform + +# Run all pre-commit hooks manually +pre-commit run --all-files +``` + +## Deployment Guide + +### Prerequisites + +Before deploying this module, ensure you have: + +1. **AWS CLI configured** with appropriate permissions +2. **Terraform installed** (>= 1.0) +3. **Required AWS resources** created (see below) + +### Required AWS Resources + +You need to create these resources before deploying: + +1. **KMS Key** for encryption +2. **S3 Bucket** for Lambda layers with the helper scripts ZIP uploaded +3. **S3 Bucket** for Lambda function code with the function ZIP uploaded +4. **AWS Organization ID** (if using AWS Organizations) +5. **IROCO2 API credentials** and endpoints + +### Step-by-Step Deployment + +1. **Configure AWS Profile**: + ```bash + aws configure --profile default + # or use existing profile + export AWS_PROFILE=default + ``` + +2. **Navigate to module directory**: + ```bash + cd lambdas/functions/iroco2-client-side-scanner/tf + ``` + +3. **Update variables**: + ```bash + # Copy example file + cp terraform.tfvars.example terraform.tfvars + + # Edit with your actual values + nano terraform.tfvars + ``` + +4. **Initialize Terraform**: + ```bash + terraform init + ``` + +5. **Plan deployment**: + ```bash + terraform plan + ``` + +6. **Deploy the module**: + ```bash + terraform apply + ``` + +7. **Verify deployment**: + ```bash + # Check outputs + terraform output + + # Verify Lambda function + aws lambda get-function --function-name IROCO2-CUR-SCRAPPER-dev + ``` + +### Example terraform.tfvars + +```hcl +# Update these with your actual values +kms_key_arn = "arn:aws:kms:eu-west-3:123456789012:key/abcd1234-..." +layer_bucket_storage = "my-lambda-layers-bucket" +layer_bucket_key = "layers/helper-scripts.zip" +aws_org_id = "o-1234567890" +cur_output_bucket_name = "iroco2-cur-output-unique-suffix" +cur_function_s3_key = "functions/cur-scanner.zip" +cur_function_s3_bucket = "my-lambda-functions-bucket" +iroco2_api_endpoint = "https://api.iroco2.example.com" +iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" +iroco2_api_key = "your-secret-api-key" + +common_tags = { + Environment = "production" + Project = "IROCO2" + Owner = "data-team" +} +``` + +### Cleanup + +To destroy the infrastructure: + +```bash +terraform destroy +``` + +**Warning**: This will delete all resources created by this module, including the S3 bucket and any data it contains. + +## Contributing + +When modifying this module: + +1. Ensure pre-commit hooks are installed and passing +2. Update variable descriptions and validation rules +3. Add new outputs for any additional resources +4. Update this README with new features or changes +5. Test the module in a development environment +6. Ensure backward compatibility when possible + +## License + +This module is part of the IROCO2 project and follows the project's licensing terms. diff --git a/lambdas/functions/iroco2-client-side-scanner/cloudwatch.tf b/lambdas/functions/iroco2-client-side-scanner/cloudwatch.tf new file mode 100644 index 0000000..d01565c --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/cloudwatch.tf @@ -0,0 +1,8 @@ +# CloudWatch Log Group for Lambda +resource "aws_cloudwatch_log_group" "lambda" { + name = var.lambda_log_group_name + retention_in_days = var.log_retention_days + kms_key_id = var.kms_key_arn + + tags = local.tags +} diff --git a/lambdas/functions/iroco2-client-side-scanner/cur_scrapper.py b/lambdas/functions/iroco2-client-side-scanner/cur_scrapper.py deleted file mode 100644 index 91bbf1b..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/cur_scrapper.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -import os - -from s3_repository_parser import S3Repository -from ec2_service import CurProcessorEC2Service -from s3_service import CurProcessorS3Service -from iroco_service import IrocoService - -class CurProcessorService: - - def __init__(self): - self.s3_repository = S3Repository() - self.cur_processor_ec2_service = CurProcessorEC2Service() - self.cur_processor_s3_service = CurProcessorS3Service() - self.iroco_service = IrocoService() - - def parsing_cur_and_send_to_api(self, bucket_name, s3_file_name): - cur_file_type = os.path.splitext(s3_file_name)[1].replace(".", "") - print(cur_file_type) - cur = self.s3_repository.read_file(bucket_name, s3_file_name) - message_parsed = [] - parsed_ec2_message = self.cur_processor_ec2_service.creating_message_from_cur(cur, cur_file_type) - print(f'EC2 messages: {parsed_ec2_message}') - parsed_s3_message = self.cur_processor_s3_service.creating_message_from_cur(cur, cur_file_type) - print(f'S3 messages: {parsed_s3_message}') - message_parsed.extend(parsed_ec2_message) - message_parsed.extend(parsed_s3_message) - self.__send_message_parsed_to_api(message_parsed) - - def __send_message_parsed_to_api(self, messages_parsed): - total_message_sent = len(messages_parsed) - correlation_id = self.iroco_service.create_scan() - - for message_parsed in messages_parsed: - message_parsed["numberOfMessageExpected"] = total_message_sent - message_parsed["correlationId"] = correlation_id - self.iroco_service.send_payload(message_parsed) - -def handler(event,context): - print(f"Event Name: {event['Records'][0]['eventName']}") - print(f"S3 Bucket Name: {event['Records'][0]['s3']['bucket']['name']}") - print(f"Object Key: {event['Records'][0]['s3']['object']['key']}") - print(f"Event: {event}") - cur_processor_service = CurProcessorService() - cur_processor_service.parsing_cur_and_send_to_api( - bucket_name=event['Records'][0]['s3']['bucket']['name'], - s3_file_name=event['Records'][0]['s3']['object']['key'], - ) diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/README.md b/lambdas/functions/iroco2-client-side-scanner/examples/basic/README.md similarity index 100% rename from lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/README.md rename to lambdas/functions/iroco2-client-side-scanner/examples/basic/README.md diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/main.tf b/lambdas/functions/iroco2-client-side-scanner/examples/basic/main.tf similarity index 79% rename from lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/main.tf rename to lambdas/functions/iroco2-client-side-scanner/examples/basic/main.tf index 8a8cf4c..aa85a80 100644 --- a/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/main.tf +++ b/lambdas/functions/iroco2-client-side-scanner/examples/basic/main.tf @@ -28,24 +28,9 @@ resource "random_id" "suffix" { byte_length = 4 } -# Upload Lambda function and layer (optional - you can use existing S3 objects) -resource "aws_s3_object" "lambda_function" { - bucket = aws_s3_bucket.lambda_artifacts.bucket - key = "functions/cur-scanner.zip" - source = "../../cur-scanner.zip" - etag = filemd5("../../cur-scanner.zip") -} - -resource "aws_s3_object" "lambda_layer" { - bucket = aws_s3_bucket.lambda_artifacts.bucket - key = "layers/helper-scripts.zip" - source = "../../helper-scripts.zip" - etag = filemd5("../../helper-scripts.zip") -} - # Use the IROCO2 Client Side Scanner module module "iroco2_scanner" { - source = "../../" + source = "../" # Required variables kms_key_arn = data.aws_kms_key.existing.arn diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/terraform.tfvars.example b/lambdas/functions/iroco2-client-side-scanner/examples/basic/terraform.tfvars.example similarity index 100% rename from lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/terraform.tfvars.example rename to lambdas/functions/iroco2-client-side-scanner/examples/basic/terraform.tfvars.example diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/variables.tf b/lambdas/functions/iroco2-client-side-scanner/examples/basic/variables.tf similarity index 100% rename from lambdas/functions/iroco2-client-side-scanner/tf/examples/basic/variables.tf rename to lambdas/functions/iroco2-client-side-scanner/examples/basic/variables.tf diff --git a/lambdas/functions/iroco2-client-side-scanner/iam.tf b/lambdas/functions/iroco2-client-side-scanner/iam.tf new file mode 100644 index 0000000..c140222 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/iam.tf @@ -0,0 +1,52 @@ +# IAM role for Lambda execution +resource "aws_iam_role" "lambda_execution" { + name = "${var.lambda_function_name}-execution-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + Action = "sts:AssumeRole" + } + ] + }) + + tags = local.tags +} + +# Attach AWS managed policy for basic Lambda execution +resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { + role = aws_iam_role.lambda_execution.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +# Custom policy for S3 and KMS access +resource "aws_iam_role_policy" "lambda_s3_kms_access" { + name = "${var.lambda_function_name}-s3-kms-policy" + role = aws_iam_role.lambda_execution.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion" + ] + Resource = "${aws_s3_bucket.cur_output.arn}/*" + }, + { + Effect = "Allow" + Action = [ + "kms:Decrypt" + ] + Resource = var.kms_key_arn + } + ] + }) +} diff --git a/lambdas/functions/iroco2-client-side-scanner/lambda.tf b/lambdas/functions/iroco2-client-side-scanner/lambda.tf new file mode 100644 index 0000000..7f80f1f --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/lambda.tf @@ -0,0 +1,70 @@ +data "archive_file" "layers" { + type = "zip" + source_dir = "${path.module}/layers" + output_path = "${path.module}/layers.zip" +} + +# Lambda Layer for helper scripts +resource "aws_lambda_layer_version" "helper_scripts" { + layer_name = local.layer_name + description = "Scripts for IROCO2 helpers" + compatible_runtimes = ["python3.9", "python3.10", "python3.11"] + + filename = "layers.zip" + +} + +# Lambda Function +resource "aws_lambda_function" "processing" { + function_name = var.lambda_function_name + description = "IROCO2 CUR Scanner - Processes CUR data for carbon impact analysis" + + role = aws_iam_role.lambda_execution.arn + handler = "cur_scrapper.handler" + runtime = "python3.11" + + timeout = var.lambda_timeout + memory_size = var.lambda_memory_size + + # Lambda layers + layers = [ + aws_lambda_layer_version.helper_scripts.arn, + var.aws_sdk_pandas_layer_arn + ] + + # Source code from S3 + s3_bucket = var.cur_function_s3_bucket + s3_key = var.cur_function_s3_key + + # Environment variables + environment { + variables = { + IROCO2_API_ENDPOINT = var.iroco2_api_endpoint + IROCO2_GATEWAY_ENDPOINT = var.iroco2_gateway_endpoint + IROCO2_API_KEY = var.iroco2_api_key + } + } + + # Logging configuration + logging_config { + log_format = "Text" + log_group = aws_cloudwatch_log_group.lambda.name + } + + tags = local.tags + + depends_on = [ + aws_iam_role_policy_attachment.lambda_basic_execution, + aws_iam_role_policy.lambda_s3_kms_access, + aws_cloudwatch_log_group.lambda + ] +} + +# Lambda permission for S3 to invoke the function +resource "aws_lambda_permission" "s3_invoke" { + statement_id = "AllowExecutionFromS3Bucket" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.processing.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.cur_output.arn +} diff --git a/lambdas/functions/iroco2-client-side-scanner/lambda.yaml b/lambdas/functions/iroco2-client-side-scanner/lambda.yaml deleted file mode 100644 index 7938821..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/lambda.yaml +++ /dev/null @@ -1,193 +0,0 @@ -AWSTemplateFormatVersion: "2010-09-09" -Description: "Template for S3 bucket with Lambda trigger on PUT events" - -Parameters: - KMSkeyARN: - Description: "Enter the KMS key to encrypt the bucket" - Type: String - LayerBucketStorage: - Description: "Enter the S3 bucket name for the lambda layer" - Type: String - LayerBucketKey: - Description: "Enter the S3 bucket key for the lambda layer" - Type: String - AWSOrgID: - Description: "Enter the AWS Org ID" - Type: String - CUROutputBucketName: - Description: "Enter the S3 bucket name for the CUR output" - Type: String - CURFunctionS3Key: - Description: "Enter the S3 bucket key for the CUR lambda function" - Type: String - CURFunctionS3Bucket: - Description: "Enter the S3 bucket name for the CUR lambda function" - Type: String - Iroco2APIEndpoint: - Description: "Enter the IroCO2 API endpoint" - Type: String - Iroco2GatewayEndpoint: - Description: "Enter the IroCO2 Gateway endpoint" - Type: String - Iroco2APIKey: - Description: "Enter the IroCO2 API token" - Type: String - LambdaLogGroupName: - Description: "Log group Name for lambda" - Type: String - -Resources: - S3bucketOutputCUR: - Type: AWS::S3::Bucket - Properties: - NotificationConfiguration: - LambdaConfigurations: - - Event: "s3:ObjectCreated:Copy" - Function: !GetAtt ProcessingFunction.Arn - Filter: - S3Key: - Rules: - - Name: suffix - Value: .parquet - - Event: "s3:ObjectCreated:Copy" - Function: !GetAtt ProcessingFunction.Arn - Filter: - S3Key: - Rules: - - Name: suffix - Value: .csv.zip - BucketName: !Ref CUROutputBucketName - BucketEncryption: - ServerSideEncryptionConfiguration: - - ServerSideEncryptionByDefault: - SSEAlgorithm: "aws:kms" - KMSMasterKeyID: !Ref KMSkeyARN - VersioningConfiguration: - Status: Suspended - PublicAccessBlockConfiguration: - BlockPublicAcls: true - BlockPublicPolicy: true - IgnorePublicAcls: true - RestrictPublicBuckets: true - Tags: - - Key: Project - Value: IROCO2 - - S3CURBucketAccessPolicy: - Type: AWS::S3::BucketPolicy - Properties: - Bucket: !Ref S3bucketOutputCUR - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - AWS: - - !Ref "AWS::AccountId" - Action: - - "s3:*" - Resource: - - !Sub "arn:aws:s3:::${CUROutputBucketName}" - - !Sub "arn:aws:s3:::${CUROutputBucketName}/*" - Condition: - Bool: - aws:SecureTransport: true - StringEquals: - aws:PrincipalOrgId: !Ref AWSOrgID - - Effect: Deny - Principal: - AWS: - - !GetAtt LambdaExecutionRole.Arn - NotAction: - - "s3:GetObject" - Resource: - - !Sub "arn:aws:s3:::${CUROutputBucketName}/*" - Condition: - Bool: - aws:SecureTransport: true - - # Lambda Function Permission - LambdaPermission: - Type: "AWS::Lambda::Permission" - Properties: - Action: "lambda:InvokeFunction" - FunctionName: !Ref ProcessingFunction - Principal: "s3.amazonaws.com" - SourceArn: !Sub "arn:aws:s3:::${CUROutputBucketName}" - - # IAM Role for Lambda - LambdaExecutionRole: - Type: "AWS::IAM::Role" - Properties: - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Service: lambda.amazonaws.com - Action: "sts:AssumeRole" - ManagedPolicyArns: - - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - Policies: - - PolicyName: S3Access - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - "s3:GetObject" - Resource: !Sub "arn:aws:s3:::${CUROutputBucketName}/*" - - Effect: Allow - Action: - - "kms:Decrypt" - Resource: !Ref KMSkeyARN - - HelperScriptLayer: - Type: AWS::Lambda::LayerVersion - Properties: - LayerName: helper-scripts - Description: Scripts for IROCO2 helpers - CompatibleRuntimes: - - python3.9 - - python3.10 - - python3.11 - Content: - S3Key: !Ref LayerBucketKey - S3Bucket: !Ref LayerBucketStorage - - # Lambda Function - ProcessingFunction: - Type: "AWS::Lambda::Function" - Properties: - FunctionName: "IROCO2-CUR-SCRAPPER" - Handler: "cur_scrapper.handler" - Role: !GetAtt LambdaExecutionRole.Arn - Layers: - - !Ref HelperScriptLayer - - !Sub "arn:aws:lambda:eu-west-3:336392948345:layer:AWSSDKPandas-Python311:18" - Runtime: python3.11 - Timeout: 900 - MemorySize: 512 - LoggingConfig: - LogGroup: !Ref LambdaLogGroupName - Environment: - Variables: - IROCO2_API_ENDPOINT: !Ref Iroco2APIEndpoint - IROCO2_GATEWAY_ENDPOINT: !Ref Iroco2GatewayEndpoint - IROCO2_API_KEY: !Ref Iroco2APIKey - Code: - S3Key: !Ref CURFunctionS3Key - S3Bucket: !Ref CURFunctionS3Bucket - - # CloudWatch Log Group for Lambda - LambdaLogGroup: - Type: "AWS::Logs::LogGroup" - Properties: - LogGroupName: !Ref LambdaLogGroupName - RetentionInDays: 14 - KmsKeyId: !Ref KMSkeyARN - -Outputs: - CURLambdaFunctionName: - Description: "Name of the created Lambda function for CUR extraction" - Value: !Ref ProcessingFunction diff --git a/lambdas/functions/iroco2-client-side-scanner/locals.tf b/lambdas/functions/iroco2-client-side-scanner/locals.tf new file mode 100644 index 0000000..6b07769 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/locals.tf @@ -0,0 +1,30 @@ +locals { + # Common tags merged with user-provided tags + tags = merge(var.common_tags, { + ManagedBy = "Terraform" + Module = "iroco2-client-side-scanner" + }) + + # Lambda layer name + layer_name = "helper-scripts" + + # S3 bucket notification events + s3_notification_events = [ + "s3:ObjectCreated:Copy" + ] + + # S3 object filters for Lambda triggers + s3_object_filters = [ + { + suffix = ".parquet" + }, + { + suffix = ".csv.zip" + } + ] +} + +# Data sources +data "aws_caller_identity" "current" {} + +data "aws_region" "current" {} diff --git a/lambdas/functions/iroco2-client-side-scanner/main.tf b/lambdas/functions/iroco2-client-side-scanner/main.tf new file mode 100644 index 0000000..e84dfc1 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/main.tf @@ -0,0 +1,5 @@ +# Main Terraform configuration for IROCO2 Client Side Scanner +# This module creates all necessary AWS resources for the CUR scanner Lambda function + +# This file serves as the main entry point and can include any additional +# configuration or data sources that don't fit in the specific resource files diff --git a/lambdas/functions/iroco2-client-side-scanner/outputs.tf b/lambdas/functions/iroco2-client-side-scanner/outputs.tf new file mode 100644 index 0000000..2a5325c --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/outputs.tf @@ -0,0 +1,59 @@ +output "lambda_function_name" { + description = "Name of the created Lambda function for CUR extraction" + value = aws_lambda_function.processing.function_name +} + +output "lambda_function_arn" { + description = "ARN of the created Lambda function" + value = aws_lambda_function.processing.arn +} + +output "lambda_function_invoke_arn" { + description = "Invoke ARN of the Lambda function" + value = aws_lambda_function.processing.invoke_arn +} + +output "s3_bucket_name" { + description = "Name of the S3 bucket for CUR output" + value = aws_s3_bucket.cur_output.bucket +} + +output "s3_bucket_arn" { + description = "ARN of the S3 bucket for CUR output" + value = aws_s3_bucket.cur_output.arn +} + +output "s3_bucket_domain_name" { + description = "Domain name of the S3 bucket" + value = aws_s3_bucket.cur_output.bucket_domain_name +} + +output "lambda_execution_role_arn" { + description = "ARN of the Lambda execution role" + value = aws_iam_role.lambda_execution.arn +} + +output "lambda_execution_role_name" { + description = "Name of the Lambda execution role" + value = aws_iam_role.lambda_execution.name +} + +output "lambda_layer_arn" { + description = "ARN of the helper scripts Lambda layer" + value = aws_lambda_layer_version.helper_scripts.arn +} + +output "lambda_layer_version" { + description = "Version of the helper scripts Lambda layer" + value = aws_lambda_layer_version.helper_scripts.version +} + +output "cloudwatch_log_group_name" { + description = "Name of the CloudWatch log group" + value = aws_cloudwatch_log_group.lambda.name +} + +output "cloudwatch_log_group_arn" { + description = "ARN of the CloudWatch log group" + value = aws_cloudwatch_log_group.lambda.arn +} diff --git a/lambdas/functions/iroco2-client-side-scanner/providers.tf b/lambdas/functions/iroco2-client-side-scanner/providers.tf new file mode 100644 index 0000000..b7861b3 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/providers.tf @@ -0,0 +1,11 @@ +provider "aws" { + region = "eu-west-3" + profile = "default" + + default_tags { + tags = { + ManagedBy = "Terraform" + Module = "iroco2-client-side-scanner" + } + } +} diff --git a/lambdas/functions/iroco2-client-side-scanner/s3.tf b/lambdas/functions/iroco2-client-side-scanner/s3.tf new file mode 100644 index 0000000..c3d79c9 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/s3.tf @@ -0,0 +1,97 @@ +# S3 bucket for CUR output +resource "aws_s3_bucket" "cur_output" { + bucket = var.cur_output_bucket_name + + tags = local.tags +} + +# S3 bucket versioning +resource "aws_s3_bucket_versioning" "cur_output" { + bucket = aws_s3_bucket.cur_output.id + versioning_configuration { + status = "Suspended" + } +} + +# S3 bucket encryption +resource "aws_s3_bucket_server_side_encryption_configuration" "cur_output" { + bucket = aws_s3_bucket.cur_output.id + + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = var.kms_key_arn + sse_algorithm = "aws:kms" + } + } +} + +# S3 bucket public access block +resource "aws_s3_bucket_public_access_block" "cur_output" { + bucket = aws_s3_bucket.cur_output.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +# S3 bucket policy +resource "aws_s3_bucket_policy" "cur_output" { + bucket = aws_s3_bucket.cur_output.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + AWS = data.aws_caller_identity.current.account_id + } + Action = "s3:*" + Resource = [ + aws_s3_bucket.cur_output.arn, + "${aws_s3_bucket.cur_output.arn}/*" + ] + Condition = { + Bool = { + "aws:SecureTransport" = "true" + } + StringEquals = { + "aws:PrincipalOrgId" = var.aws_org_id + } + } + }, + { + Effect = "Deny" + Principal = { + AWS = aws_iam_role.lambda_execution.arn + } + NotAction = "s3:GetObject" + Resource = "${aws_s3_bucket.cur_output.arn}/*" + Condition = { + Bool = { + "aws:SecureTransport" = "true" + } + } + } + ] + }) + + depends_on = [aws_s3_bucket_public_access_block.cur_output] +} + +# S3 bucket notification +resource "aws_s3_bucket_notification" "cur_output" { + bucket = aws_s3_bucket.cur_output.id + + dynamic "lambda_function" { + for_each = local.s3_object_filters + content { + lambda_function_arn = aws_lambda_function.processing.arn + events = local.s3_notification_events + filter_suffix = lambda_function.value.suffix + } + } + + depends_on = [aws_lambda_permission.s3_invoke] +} diff --git a/lambdas/functions/iroco2-client-side-scanner/terraform.tfvars.example b/lambdas/functions/iroco2-client-side-scanner/terraform.tfvars.example new file mode 100644 index 0000000..d46fcc2 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/terraform.tfvars.example @@ -0,0 +1,29 @@ +# Example Terraform variables for IROCO2 Client Side Scanner +# Copy this file to terraform.tfvars and update with your actual values + +# Required variables +kms_key_arn = "arn:aws:kms:eu-west-3:992382519845:key/02094aa9-afab-4e3c-84b2-8082380c66eb" +layer_bucket_storage = "iroco-test-sylvain-scrapper" +layer_bucket_key = "layers/helper-scripts.zip" +aws_org_id = "o-yn7toj4ob7" +cur_output_bucket_name = "iroco-cur-bucket" +cur_function_s3_key = "functions/cur-scanner.zip" +cur_function_s3_bucket = "iroco-test-sylvain-scrapper" +iroco2_api_endpoint = "https://api.iroco2.example.com" +iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" +iroco2_api_key = "your-api-key-here" + +# Optional variables (uncomment to override defaults) +# lambda_function_name = "IROCO2-CUR-SCRAPPER" +# lambda_log_group_name = "/aws/lambda/IROCO2-CUR-SCRAPPER" +# lambda_timeout = 900 +# lambda_memory_size = 512 +# log_retention_days = 14 + +# Common tags +common_tags = { + Environment = "development" + Project = "IROCO2" + Owner = "data-team" + ManagedBy = "Terraform" +} diff --git a/lambdas/functions/iroco2-client-side-scanner/test-payload.json b/lambdas/functions/iroco2-client-side-scanner/test-payload.json new file mode 100644 index 0000000..1c04e1b --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/test-payload.json @@ -0,0 +1,15 @@ +{ + "Records": [ + { + "eventName": "ObjectCreated:Copy", + "s3": { + "bucket": { + "name": "iroco-cur-bucket" + }, + "object": { + "key": "iroco/iroco_CUR_report/20250901-20251001/20250918T091302Z/iroco_CUR_report-00001.csv.zip" + } + } + } + ] +} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/README.md b/lambdas/functions/iroco2-client-side-scanner/tf/README.md index bca5eaa..0f00a2e 100644 --- a/lambdas/functions/iroco2-client-side-scanner/tf/README.md +++ b/lambdas/functions/iroco2-client-side-scanner/tf/README.md @@ -375,7 +375,7 @@ You need to create these resources before deploying: ```bash # Copy example file cp terraform.tfvars.example terraform.tfvars - + # Edit with your actual values nano terraform.tfvars ``` @@ -399,7 +399,7 @@ You need to create these resources before deploying: ```bash # Check outputs terraform output - + # Verify Lambda function aws lambda get-function --function-name IROCO2-CUR-SCRAPPER-dev ``` diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response.json b/lambdas/functions/iroco2-client-side-scanner/tf/response.json index a92b7ba..aa4f6db 100644 --- a/lambdas/functions/iroco2-client-side-scanner/tf/response.json +++ b/lambdas/functions/iroco2-client-side-scanner/tf/response.json @@ -1 +1 @@ -{"errorMessage": "Unable to import module 'cur_scrapper': No module named 'cur_scrapper'", "errorType": "Runtime.ImportModuleError", "requestId": "", "stackTrace": []} \ No newline at end of file +{"errorMessage": "Unable to import module 'cur_scrapper': No module named 'cur_scrapper'", "errorType": "Runtime.ImportModuleError", "requestId": "", "stackTrace": []} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response2.json b/lambdas/functions/iroco2-client-side-scanner/tf/response2.json index 2dd985e..8cf8a2d 100644 --- a/lambdas/functions/iroco2-client-side-scanner/tf/response2.json +++ b/lambdas/functions/iroco2-client-side-scanner/tf/response2.json @@ -1 +1 @@ -{"errorMessage": "Unable to import module 'cur_scrapper': No module named 's3_repository_parser'", "errorType": "Runtime.ImportModuleError", "requestId": "", "stackTrace": []} \ No newline at end of file +{"errorMessage": "Unable to import module 'cur_scrapper': No module named 's3_repository_parser'", "errorType": "Runtime.ImportModuleError", "requestId": "", "stackTrace": []} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response3.json b/lambdas/functions/iroco2-client-side-scanner/tf/response3.json index 36f18ec..b32be82 100644 --- a/lambdas/functions/iroco2-client-side-scanner/tf/response3.json +++ b/lambdas/functions/iroco2-client-side-scanner/tf/response3.json @@ -1 +1 @@ -{"errorMessage": "'Records'", "errorType": "KeyError", "requestId": "26fa1e0b-59b8-4faa-9bec-af6126e7244c", "stackTrace": [" File \"/var/task/cur_scrapper.py\", line 54, in handler\n print(f\"Event Name: {event['Records'][0]['eventName']}\")\n"]} \ No newline at end of file +{"errorMessage": "'Records'", "errorType": "KeyError", "requestId": "26fa1e0b-59b8-4faa-9bec-af6126e7244c", "stackTrace": [" File \"/var/task/cur_scrapper.py\", line 54, in handler\n print(f\"Event Name: {event['Records'][0]['eventName']}\")\n"]} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response4.json b/lambdas/functions/iroco2-client-side-scanner/tf/response4.json index aa1205b..e28511e 100644 --- a/lambdas/functions/iroco2-client-side-scanner/tf/response4.json +++ b/lambdas/functions/iroco2-client-side-scanner/tf/response4.json @@ -1 +1 @@ -{"errorMessage": "HTTPSConnectionPool(host='api.iroco2.example.com', port=443): Max retries exceeded with url: / (Caused by NewConnectionError(': Failed to establish a new connection: [Errno -2] Name or service not known'))", "errorType": "ConnectionError", "requestId": "f52f8f03-07bb-431d-8451-496f74e1f4d2", "stackTrace": [" File \"/var/task/cur_scrapper.py\", line 59, in handler\n cur_processor_service.parsing_cur_and_send_to_api(\n", " File \"/var/task/cur_scrapper.py\", line 42, in parsing_cur_and_send_to_api\n self.__send_message_parsed_to_api(message_parsed)\n", " File \"/var/task/cur_scrapper.py\", line 46, in __send_message_parsed_to_api\n correlation_id = self.iroco_service.create_scan()\n", " File \"/opt/python/iroco_service.py\", line 40, in create_scan\n response = requests.post(self.iroco_endpoint, headers=self.__build_header())\n", " File \"/opt/python/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n", " File \"/opt/python/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 589, in request\n resp = self.send(prep, **send_kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 703, in send\n r = adapter.send(request, **kwargs)\n", " File \"/opt/python/requests/adapters.py\", line 700, in send\n raise ConnectionError(e, request=request)\n"]} \ No newline at end of file +{"errorMessage": "HTTPSConnectionPool(host='api.iroco2.example.com', port=443): Max retries exceeded with url: / (Caused by NewConnectionError(': Failed to establish a new connection: [Errno -2] Name or service not known'))", "errorType": "ConnectionError", "requestId": "f52f8f03-07bb-431d-8451-496f74e1f4d2", "stackTrace": [" File \"/var/task/cur_scrapper.py\", line 59, in handler\n cur_processor_service.parsing_cur_and_send_to_api(\n", " File \"/var/task/cur_scrapper.py\", line 42, in parsing_cur_and_send_to_api\n self.__send_message_parsed_to_api(message_parsed)\n", " File \"/var/task/cur_scrapper.py\", line 46, in __send_message_parsed_to_api\n correlation_id = self.iroco_service.create_scan()\n", " File \"/opt/python/iroco_service.py\", line 40, in create_scan\n response = requests.post(self.iroco_endpoint, headers=self.__build_header())\n", " File \"/opt/python/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n", " File \"/opt/python/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 589, in request\n resp = self.send(prep, **send_kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 703, in send\n r = adapter.send(request, **kwargs)\n", " File \"/opt/python/requests/adapters.py\", line 700, in send\n raise ConnectionError(e, request=request)\n"]} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response5.json b/lambdas/functions/iroco2-client-side-scanner/tf/response5.json index 01f7003..312bd8a 100644 --- a/lambdas/functions/iroco2-client-side-scanner/tf/response5.json +++ b/lambdas/functions/iroco2-client-side-scanner/tf/response5.json @@ -1 +1 @@ -{"errorMessage": "HTTPSConnectionPool(host='api.iroco2.example.com', port=443): Max retries exceeded with url: / (Caused by NewConnectionError(': Failed to establish a new connection: [Errno -2] Name or service not known'))", "errorType": "ConnectionError", "requestId": "5fd110a3-d450-463d-ac4b-2bdf0e5931a6", "stackTrace": [" File \"/var/task/cur_scrapper.py\", line 60, in handler\n cur_processor_service.parsing_cur_and_send_to_api(\n", " File \"/var/task/cur_scrapper.py\", line 42, in parsing_cur_and_send_to_api\n self.__send_message_parsed_to_api(message_parsed)\n", " File \"/var/task/cur_scrapper.py\", line 46, in __send_message_parsed_to_api\n correlation_id = self.iroco_service.create_scan()\n", " File \"/opt/python/iroco_service.py\", line 40, in create_scan\n response = requests.post(self.iroco_endpoint, headers=self.__build_header())\n", " File \"/opt/python/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n", " File \"/opt/python/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 589, in request\n resp = self.send(prep, **send_kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 703, in send\n r = adapter.send(request, **kwargs)\n", " File \"/opt/python/requests/adapters.py\", line 700, in send\n raise ConnectionError(e, request=request)\n"]} \ No newline at end of file +{"errorMessage": "HTTPSConnectionPool(host='api.iroco2.example.com', port=443): Max retries exceeded with url: / (Caused by NewConnectionError(': Failed to establish a new connection: [Errno -2] Name or service not known'))", "errorType": "ConnectionError", "requestId": "5fd110a3-d450-463d-ac4b-2bdf0e5931a6", "stackTrace": [" File \"/var/task/cur_scrapper.py\", line 60, in handler\n cur_processor_service.parsing_cur_and_send_to_api(\n", " File \"/var/task/cur_scrapper.py\", line 42, in parsing_cur_and_send_to_api\n self.__send_message_parsed_to_api(message_parsed)\n", " File \"/var/task/cur_scrapper.py\", line 46, in __send_message_parsed_to_api\n correlation_id = self.iroco_service.create_scan()\n", " File \"/opt/python/iroco_service.py\", line 40, in create_scan\n response = requests.post(self.iroco_endpoint, headers=self.__build_header())\n", " File \"/opt/python/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n", " File \"/opt/python/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 589, in request\n resp = self.send(prep, **send_kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 703, in send\n r = adapter.send(request, **kwargs)\n", " File \"/opt/python/requests/adapters.py\", line 700, in send\n raise ConnectionError(e, request=request)\n"]} diff --git a/lambdas/functions/iroco2-client-side-scanner/variables.tf b/lambdas/functions/iroco2-client-side-scanner/variables.tf new file mode 100644 index 0000000..6afa01a --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/variables.tf @@ -0,0 +1,128 @@ +variable "kms_key_arn" { + description = "The KMS key ARN to encrypt the bucket and logs" + type = string + validation { + condition = can(regex("^arn:aws:kms:", var.kms_key_arn)) + error_message = "The KMS key ARN must be a valid AWS KMS key ARN." + } +} + +variable "layer_bucket_storage" { + description = "The S3 bucket name for the lambda layer" + type = string +} + +variable "layer_bucket_key" { + description = "The S3 bucket key for the lambda layer" + type = string +} + +variable "aws_org_id" { + description = "The AWS Organization ID" + type = string + validation { + condition = can(regex("^o-[a-z0-9]{10,32}$", var.aws_org_id)) + error_message = "The AWS Organization ID must be in the format o-xxxxxxxxxx." + } +} + +variable "cur_output_bucket_name" { + description = "The S3 bucket name for the CUR output" + type = string + validation { + condition = can(regex("^[a-z0-9][a-z0-9-]*[a-z0-9]$", var.cur_output_bucket_name)) + error_message = "The bucket name must be a valid S3 bucket name." + } +} + +variable "cur_function_s3_key" { + description = "The S3 bucket key for the CUR lambda function" + type = string +} + +variable "cur_function_s3_bucket" { + description = "The S3 bucket name for the CUR lambda function" + type = string +} + +variable "iroco2_api_endpoint" { + description = "The IroCO2 API endpoint" + type = string + validation { + condition = can(regex("^https?://", var.iroco2_api_endpoint)) + error_message = "The API endpoint must be a valid HTTP or HTTPS URL." + } +} + +variable "iroco2_gateway_endpoint" { + description = "The IroCO2 Gateway endpoint" + type = string + validation { + condition = can(regex("^https?://", var.iroco2_gateway_endpoint)) + error_message = "The Gateway endpoint must be a valid HTTP or HTTPS URL." + } +} + +variable "iroco2_api_key" { + description = "The IroCO2 API token" + type = string + sensitive = true +} + +variable "lambda_log_group_name" { + description = "Log group name for lambda" + type = string + default = "/aws/lambda/IROCO2-CUR-SCRAPPER" +} + +variable "lambda_function_name" { + description = "Name of the Lambda function" + type = string + default = "IROCO2-CUR-SCRAPPER" +} + +variable "lambda_timeout" { + description = "Lambda function timeout in seconds" + type = number + default = 900 + validation { + condition = var.lambda_timeout >= 1 && var.lambda_timeout <= 900 + error_message = "Lambda timeout must be between 1 and 900 seconds." + } +} + +variable "lambda_memory_size" { + description = "Lambda function memory size in MB" + type = number + default = 512 + validation { + condition = var.lambda_memory_size >= 128 && var.lambda_memory_size <= 10240 + error_message = "Lambda memory size must be between 128 and 10240 MB." + } +} + +variable "log_retention_days" { + description = "CloudWatch log retention in days" + type = number + default = 14 + validation { + condition = contains([ + 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653 + ], var.log_retention_days) + error_message = "Log retention days must be a valid CloudWatch Logs retention value." + } +} + +variable "common_tags" { + description = "Common tags to apply to all resources" + type = map(string) + default = { + Project = "IROCO2" + } +} + +variable "aws_sdk_pandas_layer_arn" { + description = "ARN of the AWS SDK Pandas layer" + type = string + default = "arn:aws:lambda:eu-west-3:336392948345:layer:AWSSDKPandas-Python311:18" +} diff --git a/lambdas/functions/iroco2-client-side-scanner/versions.tf b/lambdas/functions/iroco2-client-side-scanner/versions.tf new file mode 100644 index 0000000..ddfcb0e --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } +} From 2683660a48a79ed8070fd4d76b5f70ff729058e3 Mon Sep 17 00:00:00 2001 From: sressier Date: Fri, 26 Sep 2025 10:43:32 +0200 Subject: [PATCH 4/9] remove tf folder --- .../iroco2-client-side-scanner/tf/.gitignore | 22 - .../tf/.pre-commit-config.yaml | 38 -- .../iroco2-client-side-scanner/tf/.tflint.hcl | 95 ---- .../iroco2-client-side-scanner/tf/README.md | 452 ------------------ .../tf/cloudwatch.tf | 8 - .../iroco2-client-side-scanner/tf/iam.tf | 52 -- .../iroco2-client-side-scanner/tf/lambda.tf | 67 --- .../iroco2-client-side-scanner/tf/locals.tf | 30 -- .../iroco2-client-side-scanner/tf/main.tf | 5 - .../iroco2-client-side-scanner/tf/outputs.tf | 59 --- .../tf/providers.tf | 11 - .../tf/response.json | 1 - .../tf/response2.json | 1 - .../tf/response3.json | 1 - .../tf/response4.json | 1 - .../tf/response5.json | 1 - .../iroco2-client-side-scanner/tf/s3.tf | 97 ---- .../tf/terraform.tfvars.example | 29 -- .../tf/test-payload.json | 15 - .../tf/variables.tf | 128 ----- .../iroco2-client-side-scanner/tf/versions.tf | 10 - 21 files changed, 1123 deletions(-) delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/.gitignore delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/.pre-commit-config.yaml delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/.tflint.hcl delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/README.md delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/cloudwatch.tf delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/iam.tf delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/lambda.tf delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/locals.tf delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/main.tf delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/outputs.tf delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/providers.tf delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/response.json delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/response2.json delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/response3.json delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/response4.json delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/response5.json delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/s3.tf delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/terraform.tfvars.example delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/test-payload.json delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/variables.tf delete mode 100644 lambdas/functions/iroco2-client-side-scanner/tf/versions.tf diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/.gitignore b/lambdas/functions/iroco2-client-side-scanner/tf/.gitignore deleted file mode 100644 index fe5ac31..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/.gitignore +++ /dev/null @@ -1,22 +0,0 @@ -# Terraform files -*.tfstate -*.tfstate.* -*.tfvars -!terraform.tfvars.example -.terraform/ -.terraform.lock.hcl -crash.log -crash.*.log - -# IDE files -.vscode/ -.idea/ -*.swp -*.swo - -# OS files -.DS_Store -Thumbs.db - -# Pre-commit -.secrets.baseline diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/.pre-commit-config.yaml b/lambdas/functions/iroco2-client-side-scanner/tf/.pre-commit-config.yaml deleted file mode 100644 index 3721e0a..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/.pre-commit-config.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Simplified pre-commit configuration for IROCO2 Client Side Scanner Terraform module -# This configuration includes only essential checks that work without additional tool installation - -repos: - # General hooks - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - - id: check-added-large-files - - id: check-case-conflict - - id: check-merge-conflict - - # Terraform hooks (minimal set) - - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.88.0 - hooks: - - id: terraform_fmt - description: Rewrites all Terraform configuration files to a canonical format - - id: terraform_validate - description: Validates all Terraform configuration files - args: - - --hook-config=--retry-once-with-cleanup=true - -# Configuration for specific hooks -default_language_version: - python: python3 - -# Global excludes -exclude: | - (?x)^( - \.terraform/.*| - \.terraform\.lock\.hcl| - terraform\.tfstate.*| - \.git/.* - )$ diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/.tflint.hcl b/lambdas/functions/iroco2-client-side-scanner/tf/.tflint.hcl deleted file mode 100644 index 924d8ff..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/.tflint.hcl +++ /dev/null @@ -1,95 +0,0 @@ -# TFLint configuration for IROCO2 Client Side Scanner Terraform module - -config { - # Enable all rules by default - disabled_by_default = false - - # Plugin directory - plugin_dir = "~/.tflint.d/plugins" - - # Call module inspection - call_module_type = "all" -} - -# AWS plugin configuration -plugin "aws" { - enabled = true - version = "0.29.0" - source = "github.com/terraform-linters/tflint-ruleset-aws" - - # Deep checking requires AWS credentials - deep_check = false -} - -# Terraform plugin configuration -plugin "terraform" { - enabled = true - preset = "recommended" -} - -# Rule configurations -rule "terraform_deprecated_interpolation" { - enabled = true -} - -rule "terraform_deprecated_index" { - enabled = true -} - -rule "terraform_unused_declarations" { - enabled = true -} - -rule "terraform_comment_syntax" { - enabled = true -} - -rule "terraform_documented_outputs" { - enabled = true -} - -rule "terraform_documented_variables" { - enabled = true -} - -rule "terraform_typed_variables" { - enabled = true -} - -rule "terraform_module_pinned_source" { - enabled = true -} - -rule "terraform_naming_convention" { - enabled = true - format = "snake_case" -} - -rule "terraform_required_version" { - enabled = true -} - -rule "terraform_required_providers" { - enabled = true -} - -rule "terraform_standard_module_structure" { - enabled = true -} - -# AWS specific rules -rule "aws_instance_invalid_type" { - enabled = true -} - -rule "aws_lambda_function_invalid_runtime" { - enabled = true -} - -rule "aws_s3_bucket_invalid_region" { - enabled = true -} - -rule "aws_iam_policy_invalid_policy" { - enabled = true -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/README.md b/lambdas/functions/iroco2-client-side-scanner/tf/README.md deleted file mode 100644 index 0f00a2e..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/README.md +++ /dev/null @@ -1,452 +0,0 @@ -# IROCO2 Client Side Scanner Terraform Module - -[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[![Terraform](https://img.shields.io/badge/Terraform-%3E%3D1.0-623CE4)](https://www.terraform.io/) -[![AWS](https://img.shields.io/badge/AWS-%3E%3D5.0-FF9900)](https://aws.amazon.com/) - -This Terraform module creates the AWS infrastructure for the IROCO2 Client Side Scanner, which processes Cost and Usage Report (CUR) data for carbon impact analysis. - -> **Open Source Ready**: This module is designed for easy integration into open source projects and can be used by anyone to analyze AWS carbon footprint data. - -## Quick Start - -Want to get started quickly? Check out our [basic example](examples/basic/) which includes: - -- πŸ“‹ **Step-by-step setup guide** -- πŸ”§ **Pre-configured variables** -- πŸ“– **Detailed documentation** -- πŸš€ **Ready-to-deploy configuration** - -```bash -# Clone the example -cp -r examples/basic my-iroco2-deployment -cd my-iroco2-deployment - -# Configure your settings -cp terraform.tfvars.example terraform.tfvars -# Edit terraform.tfvars with your AWS details - -# Deploy -terraform init && terraform apply -``` - -## Overview - -The module provisions the following AWS resources: - -- **S3 Bucket**: Encrypted storage for CUR output with event notifications -- **Lambda Function**: Python 3.11 runtime for processing CUR data -- **Lambda Layer**: Custom helper scripts for IROCO2 functionality -- **IAM Role & Policies**: Execution permissions for Lambda with least privilege access -- **CloudWatch Log Group**: Centralized logging with KMS encryption -- **S3 Bucket Policy**: Organization-based access control -- **Lambda Permissions**: S3 trigger configuration - -## Architecture - -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ S3 Bucket │───▢│ Lambda Function │───▢│ IROCO2 API β”‚ -β”‚ (CUR Output) β”‚ β”‚ (CUR Scanner) β”‚ β”‚ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ β”‚ - β”‚ β–Ό - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ β”‚ CloudWatch Logs β”‚ - β”‚ β”‚ β”‚ - β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ KMS Key β”‚ -β”‚ (Encryption) β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -## Usage - -### Basic Usage - -```hcl -module "iroco2_client_side_scanner" { - source = "./lambdas/functions/iroco2-client-side-scanner/tf" - - # Required variables - kms_key_arn = "arn:aws:kms:eu-west-3:123456789012:key/12345678-1234-1234-1234-123456789012" - layer_bucket_storage = "my-lambda-layers-bucket" - layer_bucket_key = "layers/helper-scripts.zip" - aws_org_id = "o-1234567890" - cur_output_bucket_name = "my-cur-output-bucket" - cur_function_s3_key = "functions/cur-scanner.zip" - cur_function_s3_bucket = "my-lambda-functions-bucket" - iroco2_api_endpoint = "https://api.iroco2.example.com" - iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" - iroco2_api_key = "your-api-key-here" - - # Optional variables with defaults - lambda_function_name = "IROCO2-CUR-SCRAPPER" - lambda_log_group_name = "/aws/lambda/IROCO2-CUR-SCRAPPER" - lambda_timeout = 900 - lambda_memory_size = 512 - log_retention_days = 14 - - common_tags = { - Environment = "production" - Project = "IROCO2" - Owner = "data-team" - } -} -``` - -### Advanced Usage with Custom Configuration - -```hcl -module "iroco2_client_side_scanner" { - source = "./lambdas/functions/iroco2-client-side-scanner/tf" - - # Required variables - kms_key_arn = data.aws_kms_key.main.arn - layer_bucket_storage = aws_s3_bucket.lambda_layers.bucket - layer_bucket_key = "layers/helper-scripts-v2.zip" - aws_org_id = var.organization_id - cur_output_bucket_name = "${var.environment}-cur-output-${random_id.bucket_suffix.hex}" - cur_function_s3_key = "functions/cur-scanner-${var.version}.zip" - cur_function_s3_bucket = aws_s3_bucket.lambda_functions.bucket - iroco2_api_endpoint = var.iroco2_api_endpoint - iroco2_gateway_endpoint = var.iroco2_gateway_endpoint - iroco2_api_key = data.aws_secretsmanager_secret_version.iroco2_api_key.secret_string - - # Custom configuration - lambda_function_name = "${var.environment}-iroco2-cur-scanner" - lambda_log_group_name = "/aws/lambda/${var.environment}-iroco2-cur-scanner" - lambda_timeout = 600 - lambda_memory_size = 1024 - log_retention_days = 30 - - common_tags = merge(var.common_tags, { - Environment = var.environment - Module = "iroco2-client-side-scanner" - }) -} -``` - -## Requirements - -| Name | Version | -|------|---------| -| terraform | >= 1.0 | -| aws | >= 5.0 | - -## Providers - -| Name | Version | -|------|---------| -| aws | >= 5.0 | - -## Resources - -| Name | Type | -|------|------| -| aws_s3_bucket.cur_output | resource | -| aws_s3_bucket_versioning.cur_output | resource | -| aws_s3_bucket_server_side_encryption_configuration.cur_output | resource | -| aws_s3_bucket_public_access_block.cur_output | resource | -| aws_s3_bucket_policy.cur_output | resource | -| aws_s3_bucket_notification.cur_output | resource | -| aws_lambda_function.processing | resource | -| aws_lambda_layer_version.helper_scripts | resource | -| aws_lambda_permission.s3_invoke | resource | -| aws_iam_role.lambda_execution | resource | -| aws_iam_role_policy_attachment.lambda_basic_execution | resource | -| aws_iam_role_policy.lambda_s3_kms_access | resource | -| aws_cloudwatch_log_group.lambda | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| kms_key_arn | The KMS key ARN to encrypt the bucket and logs | `string` | n/a | yes | -| layer_bucket_storage | The S3 bucket name for the lambda layer | `string` | n/a | yes | -| layer_bucket_key | The S3 bucket key for the lambda layer | `string` | n/a | yes | -| aws_org_id | The AWS Organization ID | `string` | n/a | yes | -| cur_output_bucket_name | The S3 bucket name for the CUR output | `string` | n/a | yes | -| cur_function_s3_key | The S3 bucket key for the CUR lambda function | `string` | n/a | yes | -| cur_function_s3_bucket | The S3 bucket name for the CUR lambda function | `string` | n/a | yes | -| iroco2_api_endpoint | The IroCO2 API endpoint | `string` | n/a | yes | -| iroco2_gateway_endpoint | The IroCO2 Gateway endpoint | `string` | n/a | yes | -| iroco2_api_key | The IroCO2 API token | `string` | n/a | yes | -| lambda_function_name | Name of the Lambda function | `string` | `"IROCO2-CUR-SCRAPPER"` | no | -| lambda_log_group_name | Log group name for lambda | `string` | `"/aws/lambda/IROCO2-CUR-SCRAPPER"` | no | -| lambda_timeout | Lambda function timeout in seconds | `number` | `900` | no | -| lambda_memory_size | Lambda function memory size in MB | `number` | `512` | no | -| log_retention_days | CloudWatch log retention in days | `number` | `14` | no | -| common_tags | Common tags to apply to all resources | `map(string)` | `{"Project": "IROCO2"}` | no | -| aws_sdk_pandas_layer_arn | ARN of the AWS SDK Pandas layer | `string` | `"arn:aws:lambda:eu-west-3:336392948345:layer:AWSSDKPandas-Python311:18"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| lambda_function_name | Name of the created Lambda function for CUR extraction | -| lambda_function_arn | ARN of the created Lambda function | -| lambda_function_invoke_arn | Invoke ARN of the Lambda function | -| s3_bucket_name | Name of the S3 bucket for CUR output | -| s3_bucket_arn | ARN of the S3 bucket for CUR output | -| s3_bucket_domain_name | Domain name of the S3 bucket | -| lambda_execution_role_arn | ARN of the Lambda execution role | -| lambda_execution_role_name | Name of the Lambda execution role | -| lambda_layer_arn | ARN of the helper scripts Lambda layer | -| lambda_layer_version | Version of the helper scripts Lambda layer | -| cloudwatch_log_group_name | Name of the CloudWatch log group | -| cloudwatch_log_group_arn | ARN of the CloudWatch log group | - -## Features - -### Security - -- **KMS Encryption**: All data at rest is encrypted using customer-managed KMS keys -- **IAM Least Privilege**: Lambda execution role has minimal required permissions -- **Organization Access Control**: S3 bucket policy restricts access to organization members -- **Secure Transport**: All S3 operations require HTTPS - -### Monitoring & Logging - -- **CloudWatch Integration**: Centralized logging with configurable retention -- **Structured Logging**: Lambda function uses structured logging format -- **Error Tracking**: Failed executions are logged for troubleshooting - -### Scalability - -- **Configurable Resources**: Memory and timeout can be adjusted based on workload -- **Layer Management**: Helper scripts are packaged as reusable Lambda layers -- **Event-Driven**: S3 events automatically trigger processing - -## Triggers - -The Lambda function is automatically triggered by S3 events: - -- **Event Type**: `s3:ObjectCreated:Copy` -- **File Filters**: - - Files ending with `.parquet` - - Files ending with `.csv.zip` - -## Environment Variables - -The Lambda function receives the following environment variables: - -- `IROCO2_API_ENDPOINT`: API endpoint for IROCO2 service -- `IROCO2_GATEWAY_ENDPOINT`: Gateway endpoint for IROCO2 service -- `IROCO2_API_KEY`: Authentication token for IROCO2 API - -## Migration from CloudFormation - -This module is designed as a direct replacement for the CloudFormation template in `lambda.yaml`. Key improvements include: - -1. **Better Resource Management**: Terraform state tracking -2. **Input Validation**: Variable validation rules -3. **Modular Design**: Reusable across environments -4. **Enhanced Documentation**: Comprehensive usage examples -5. **Flexible Configuration**: More customization options - -## Troubleshooting - -### Common Issues - -1. **KMS Key Access**: Ensure the KMS key policy allows Lambda service access -2. **S3 Permissions**: Verify the Lambda execution role has GetObject permissions -3. **Layer Compatibility**: Ensure the helper scripts layer is compatible with Python 3.11 -4. **Organization ID**: Verify the AWS Organization ID format is correct - -### Debugging - -Check CloudWatch logs for Lambda execution details: - -```bash -aws logs describe-log-groups --log-group-name-prefix "/aws/lambda/IROCO2-CUR-SCRAPPER" -aws logs get-log-events --log-group-name "/aws/lambda/IROCO2-CUR-SCRAPPER" --log-stream-name "LATEST" -``` - -## Development Setup - -### Prerequisites - -Before working with this module, ensure you have the following tools installed: - -- [Terraform](https://www.terraform.io/downloads.html) >= 1.0 -- [pre-commit](https://pre-commit.com/#installation) -- [TFLint](https://github.com/terraform-linters/tflint#installation) -- [tfsec](https://github.com/aquasecurity/tfsec#installation) -- [Checkov](https://www.checkov.io/1.Welcome/Quick%20Start.html) (optional, installed via pip) -- [terraform-docs](https://terraform-docs.io/user-guide/installation/) (optional) - -### Setup Pre-commit Hooks - -This module includes pre-commit configuration that automatically validates your changes: - -```bash -# Navigate to the module directory -cd lambdas/functions/iroco2-client-side-scanner/tf - -# Install pre-commit hooks (one-time setup) -pre-commit install - -# Optional: Run hooks on all files to test setup -pre-commit run --all-files -``` - -### What the Pre-commit Hooks Do - -The pre-commit configuration automatically runs the following checks on every commit: - -1. **General Checks**: - - Remove trailing whitespace - - Fix end-of-file formatting - - Validate YAML and JSON files - - Check for large files and merge conflicts - -2. **Terraform Validation**: - - Format Terraform files (`terraform fmt`) - - Validate Terraform syntax (`terraform validate`) - - Run TFLint for best practices - - Security scanning with tfsec and Checkov - - Update documentation automatically - -3. **Security**: - - Scan for secrets and sensitive data - - Check for security misconfigurations - -### Manual Commands - -If you need to run checks manually: - -```bash -# Format Terraform files -terraform fmt -recursive - -# Validate Terraform configuration -terraform validate - -# Run TFLint -tflint - -# Run security checks -tfsec . -checkov -f . --framework terraform - -# Run all pre-commit hooks manually -pre-commit run --all-files -``` - -## Deployment Guide - -### Prerequisites - -Before deploying this module, ensure you have: - -1. **AWS CLI configured** with appropriate permissions -2. **Terraform installed** (>= 1.0) -3. **Required AWS resources** created (see below) - -### Required AWS Resources - -You need to create these resources before deploying: - -1. **KMS Key** for encryption -2. **S3 Bucket** for Lambda layers with the helper scripts ZIP uploaded -3. **S3 Bucket** for Lambda function code with the function ZIP uploaded -4. **AWS Organization ID** (if using AWS Organizations) -5. **IROCO2 API credentials** and endpoints - -### Step-by-Step Deployment - -1. **Configure AWS Profile**: - ```bash - aws configure --profile default - # or use existing profile - export AWS_PROFILE=default - ``` - -2. **Navigate to module directory**: - ```bash - cd lambdas/functions/iroco2-client-side-scanner/tf - ``` - -3. **Update variables**: - ```bash - # Copy example file - cp terraform.tfvars.example terraform.tfvars - - # Edit with your actual values - nano terraform.tfvars - ``` - -4. **Initialize Terraform**: - ```bash - terraform init - ``` - -5. **Plan deployment**: - ```bash - terraform plan - ``` - -6. **Deploy the module**: - ```bash - terraform apply - ``` - -7. **Verify deployment**: - ```bash - # Check outputs - terraform output - - # Verify Lambda function - aws lambda get-function --function-name IROCO2-CUR-SCRAPPER-dev - ``` - -### Example terraform.tfvars - -```hcl -# Update these with your actual values -kms_key_arn = "arn:aws:kms:eu-west-3:123456789012:key/abcd1234-..." -layer_bucket_storage = "my-lambda-layers-bucket" -layer_bucket_key = "layers/helper-scripts.zip" -aws_org_id = "o-1234567890" -cur_output_bucket_name = "iroco2-cur-output-unique-suffix" -cur_function_s3_key = "functions/cur-scanner.zip" -cur_function_s3_bucket = "my-lambda-functions-bucket" -iroco2_api_endpoint = "https://api.iroco2.example.com" -iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" -iroco2_api_key = "your-secret-api-key" - -common_tags = { - Environment = "production" - Project = "IROCO2" - Owner = "data-team" -} -``` - -### Cleanup - -To destroy the infrastructure: - -```bash -terraform destroy -``` - -**Warning**: This will delete all resources created by this module, including the S3 bucket and any data it contains. - -## Contributing - -When modifying this module: - -1. Ensure pre-commit hooks are installed and passing -2. Update variable descriptions and validation rules -3. Add new outputs for any additional resources -4. Update this README with new features or changes -5. Test the module in a development environment -6. Ensure backward compatibility when possible - -## License - -This module is part of the IROCO2 project and follows the project's licensing terms. diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/cloudwatch.tf b/lambdas/functions/iroco2-client-side-scanner/tf/cloudwatch.tf deleted file mode 100644 index d01565c..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/cloudwatch.tf +++ /dev/null @@ -1,8 +0,0 @@ -# CloudWatch Log Group for Lambda -resource "aws_cloudwatch_log_group" "lambda" { - name = var.lambda_log_group_name - retention_in_days = var.log_retention_days - kms_key_id = var.kms_key_arn - - tags = local.tags -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/iam.tf b/lambdas/functions/iroco2-client-side-scanner/tf/iam.tf deleted file mode 100644 index c140222..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/iam.tf +++ /dev/null @@ -1,52 +0,0 @@ -# IAM role for Lambda execution -resource "aws_iam_role" "lambda_execution" { - name = "${var.lambda_function_name}-execution-role" - - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Principal = { - Service = "lambda.amazonaws.com" - } - Action = "sts:AssumeRole" - } - ] - }) - - tags = local.tags -} - -# Attach AWS managed policy for basic Lambda execution -resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { - role = aws_iam_role.lambda_execution.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" -} - -# Custom policy for S3 and KMS access -resource "aws_iam_role_policy" "lambda_s3_kms_access" { - name = "${var.lambda_function_name}-s3-kms-policy" - role = aws_iam_role.lambda_execution.id - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = [ - "s3:GetObject", - "s3:GetObjectVersion" - ] - Resource = "${aws_s3_bucket.cur_output.arn}/*" - }, - { - Effect = "Allow" - Action = [ - "kms:Decrypt" - ] - Resource = var.kms_key_arn - } - ] - }) -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/lambda.tf b/lambdas/functions/iroco2-client-side-scanner/tf/lambda.tf deleted file mode 100644 index 7214082..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/lambda.tf +++ /dev/null @@ -1,67 +0,0 @@ -# Lambda Layer for helper scripts -resource "aws_lambda_layer_version" "helper_scripts" { - layer_name = local.layer_name - description = "Scripts for IROCO2 helpers" - compatible_runtimes = ["python3.9", "python3.10", "python3.11"] - - s3_bucket = var.layer_bucket_storage - s3_key = var.layer_bucket_key - - # Layer will be updated when S3 object changes - # source_code_hash can be added if you want to track local changes -} - -# Lambda Function -resource "aws_lambda_function" "processing" { - function_name = var.lambda_function_name - description = "IROCO2 CUR Scanner - Processes CUR data for carbon impact analysis" - - role = aws_iam_role.lambda_execution.arn - handler = "cur_scrapper.handler" - runtime = "python3.11" - - timeout = var.lambda_timeout - memory_size = var.lambda_memory_size - - # Lambda layers - layers = [ - aws_lambda_layer_version.helper_scripts.arn, - var.aws_sdk_pandas_layer_arn - ] - - # Source code from S3 - s3_bucket = var.cur_function_s3_bucket - s3_key = var.cur_function_s3_key - - # Environment variables - environment { - variables = { - IROCO2_API_ENDPOINT = var.iroco2_api_endpoint - IROCO2_GATEWAY_ENDPOINT = var.iroco2_gateway_endpoint - IROCO2_API_KEY = var.iroco2_api_key - } - } - - # Logging configuration - logging_config { - log_format = "Text" - log_group = aws_cloudwatch_log_group.lambda.name - } - - tags = local.tags - - depends_on = [ - aws_iam_role_policy_attachment.lambda_basic_execution, - aws_iam_role_policy.lambda_s3_kms_access, - aws_cloudwatch_log_group.lambda - ] -} - -# Lambda permission for S3 to invoke the function -resource "aws_lambda_permission" "s3_invoke" { - statement_id = "AllowExecutionFromS3Bucket" - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.processing.function_name - principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.cur_output.arn -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/locals.tf b/lambdas/functions/iroco2-client-side-scanner/tf/locals.tf deleted file mode 100644 index 6b07769..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/locals.tf +++ /dev/null @@ -1,30 +0,0 @@ -locals { - # Common tags merged with user-provided tags - tags = merge(var.common_tags, { - ManagedBy = "Terraform" - Module = "iroco2-client-side-scanner" - }) - - # Lambda layer name - layer_name = "helper-scripts" - - # S3 bucket notification events - s3_notification_events = [ - "s3:ObjectCreated:Copy" - ] - - # S3 object filters for Lambda triggers - s3_object_filters = [ - { - suffix = ".parquet" - }, - { - suffix = ".csv.zip" - } - ] -} - -# Data sources -data "aws_caller_identity" "current" {} - -data "aws_region" "current" {} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/main.tf b/lambdas/functions/iroco2-client-side-scanner/tf/main.tf deleted file mode 100644 index e84dfc1..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/main.tf +++ /dev/null @@ -1,5 +0,0 @@ -# Main Terraform configuration for IROCO2 Client Side Scanner -# This module creates all necessary AWS resources for the CUR scanner Lambda function - -# This file serves as the main entry point and can include any additional -# configuration or data sources that don't fit in the specific resource files diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/outputs.tf b/lambdas/functions/iroco2-client-side-scanner/tf/outputs.tf deleted file mode 100644 index 2a5325c..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/outputs.tf +++ /dev/null @@ -1,59 +0,0 @@ -output "lambda_function_name" { - description = "Name of the created Lambda function for CUR extraction" - value = aws_lambda_function.processing.function_name -} - -output "lambda_function_arn" { - description = "ARN of the created Lambda function" - value = aws_lambda_function.processing.arn -} - -output "lambda_function_invoke_arn" { - description = "Invoke ARN of the Lambda function" - value = aws_lambda_function.processing.invoke_arn -} - -output "s3_bucket_name" { - description = "Name of the S3 bucket for CUR output" - value = aws_s3_bucket.cur_output.bucket -} - -output "s3_bucket_arn" { - description = "ARN of the S3 bucket for CUR output" - value = aws_s3_bucket.cur_output.arn -} - -output "s3_bucket_domain_name" { - description = "Domain name of the S3 bucket" - value = aws_s3_bucket.cur_output.bucket_domain_name -} - -output "lambda_execution_role_arn" { - description = "ARN of the Lambda execution role" - value = aws_iam_role.lambda_execution.arn -} - -output "lambda_execution_role_name" { - description = "Name of the Lambda execution role" - value = aws_iam_role.lambda_execution.name -} - -output "lambda_layer_arn" { - description = "ARN of the helper scripts Lambda layer" - value = aws_lambda_layer_version.helper_scripts.arn -} - -output "lambda_layer_version" { - description = "Version of the helper scripts Lambda layer" - value = aws_lambda_layer_version.helper_scripts.version -} - -output "cloudwatch_log_group_name" { - description = "Name of the CloudWatch log group" - value = aws_cloudwatch_log_group.lambda.name -} - -output "cloudwatch_log_group_arn" { - description = "ARN of the CloudWatch log group" - value = aws_cloudwatch_log_group.lambda.arn -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/providers.tf b/lambdas/functions/iroco2-client-side-scanner/tf/providers.tf deleted file mode 100644 index b7861b3..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/providers.tf +++ /dev/null @@ -1,11 +0,0 @@ -provider "aws" { - region = "eu-west-3" - profile = "default" - - default_tags { - tags = { - ManagedBy = "Terraform" - Module = "iroco2-client-side-scanner" - } - } -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response.json b/lambdas/functions/iroco2-client-side-scanner/tf/response.json deleted file mode 100644 index aa4f6db..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/response.json +++ /dev/null @@ -1 +0,0 @@ -{"errorMessage": "Unable to import module 'cur_scrapper': No module named 'cur_scrapper'", "errorType": "Runtime.ImportModuleError", "requestId": "", "stackTrace": []} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response2.json b/lambdas/functions/iroco2-client-side-scanner/tf/response2.json deleted file mode 100644 index 8cf8a2d..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/response2.json +++ /dev/null @@ -1 +0,0 @@ -{"errorMessage": "Unable to import module 'cur_scrapper': No module named 's3_repository_parser'", "errorType": "Runtime.ImportModuleError", "requestId": "", "stackTrace": []} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response3.json b/lambdas/functions/iroco2-client-side-scanner/tf/response3.json deleted file mode 100644 index b32be82..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/response3.json +++ /dev/null @@ -1 +0,0 @@ -{"errorMessage": "'Records'", "errorType": "KeyError", "requestId": "26fa1e0b-59b8-4faa-9bec-af6126e7244c", "stackTrace": [" File \"/var/task/cur_scrapper.py\", line 54, in handler\n print(f\"Event Name: {event['Records'][0]['eventName']}\")\n"]} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response4.json b/lambdas/functions/iroco2-client-side-scanner/tf/response4.json deleted file mode 100644 index e28511e..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/response4.json +++ /dev/null @@ -1 +0,0 @@ -{"errorMessage": "HTTPSConnectionPool(host='api.iroco2.example.com', port=443): Max retries exceeded with url: / (Caused by NewConnectionError(': Failed to establish a new connection: [Errno -2] Name or service not known'))", "errorType": "ConnectionError", "requestId": "f52f8f03-07bb-431d-8451-496f74e1f4d2", "stackTrace": [" File \"/var/task/cur_scrapper.py\", line 59, in handler\n cur_processor_service.parsing_cur_and_send_to_api(\n", " File \"/var/task/cur_scrapper.py\", line 42, in parsing_cur_and_send_to_api\n self.__send_message_parsed_to_api(message_parsed)\n", " File \"/var/task/cur_scrapper.py\", line 46, in __send_message_parsed_to_api\n correlation_id = self.iroco_service.create_scan()\n", " File \"/opt/python/iroco_service.py\", line 40, in create_scan\n response = requests.post(self.iroco_endpoint, headers=self.__build_header())\n", " File \"/opt/python/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n", " File \"/opt/python/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 589, in request\n resp = self.send(prep, **send_kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 703, in send\n r = adapter.send(request, **kwargs)\n", " File \"/opt/python/requests/adapters.py\", line 700, in send\n raise ConnectionError(e, request=request)\n"]} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/response5.json b/lambdas/functions/iroco2-client-side-scanner/tf/response5.json deleted file mode 100644 index 312bd8a..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/response5.json +++ /dev/null @@ -1 +0,0 @@ -{"errorMessage": "HTTPSConnectionPool(host='api.iroco2.example.com', port=443): Max retries exceeded with url: / (Caused by NewConnectionError(': Failed to establish a new connection: [Errno -2] Name or service not known'))", "errorType": "ConnectionError", "requestId": "5fd110a3-d450-463d-ac4b-2bdf0e5931a6", "stackTrace": [" File \"/var/task/cur_scrapper.py\", line 60, in handler\n cur_processor_service.parsing_cur_and_send_to_api(\n", " File \"/var/task/cur_scrapper.py\", line 42, in parsing_cur_and_send_to_api\n self.__send_message_parsed_to_api(message_parsed)\n", " File \"/var/task/cur_scrapper.py\", line 46, in __send_message_parsed_to_api\n correlation_id = self.iroco_service.create_scan()\n", " File \"/opt/python/iroco_service.py\", line 40, in create_scan\n response = requests.post(self.iroco_endpoint, headers=self.__build_header())\n", " File \"/opt/python/requests/api.py\", line 115, in post\n return request(\"post\", url, data=data, json=json, **kwargs)\n", " File \"/opt/python/requests/api.py\", line 59, in request\n return session.request(method=method, url=url, **kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 589, in request\n resp = self.send(prep, **send_kwargs)\n", " File \"/opt/python/requests/sessions.py\", line 703, in send\n r = adapter.send(request, **kwargs)\n", " File \"/opt/python/requests/adapters.py\", line 700, in send\n raise ConnectionError(e, request=request)\n"]} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/s3.tf b/lambdas/functions/iroco2-client-side-scanner/tf/s3.tf deleted file mode 100644 index c3d79c9..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/s3.tf +++ /dev/null @@ -1,97 +0,0 @@ -# S3 bucket for CUR output -resource "aws_s3_bucket" "cur_output" { - bucket = var.cur_output_bucket_name - - tags = local.tags -} - -# S3 bucket versioning -resource "aws_s3_bucket_versioning" "cur_output" { - bucket = aws_s3_bucket.cur_output.id - versioning_configuration { - status = "Suspended" - } -} - -# S3 bucket encryption -resource "aws_s3_bucket_server_side_encryption_configuration" "cur_output" { - bucket = aws_s3_bucket.cur_output.id - - rule { - apply_server_side_encryption_by_default { - kms_master_key_id = var.kms_key_arn - sse_algorithm = "aws:kms" - } - } -} - -# S3 bucket public access block -resource "aws_s3_bucket_public_access_block" "cur_output" { - bucket = aws_s3_bucket.cur_output.id - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -# S3 bucket policy -resource "aws_s3_bucket_policy" "cur_output" { - bucket = aws_s3_bucket.cur_output.id - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Principal = { - AWS = data.aws_caller_identity.current.account_id - } - Action = "s3:*" - Resource = [ - aws_s3_bucket.cur_output.arn, - "${aws_s3_bucket.cur_output.arn}/*" - ] - Condition = { - Bool = { - "aws:SecureTransport" = "true" - } - StringEquals = { - "aws:PrincipalOrgId" = var.aws_org_id - } - } - }, - { - Effect = "Deny" - Principal = { - AWS = aws_iam_role.lambda_execution.arn - } - NotAction = "s3:GetObject" - Resource = "${aws_s3_bucket.cur_output.arn}/*" - Condition = { - Bool = { - "aws:SecureTransport" = "true" - } - } - } - ] - }) - - depends_on = [aws_s3_bucket_public_access_block.cur_output] -} - -# S3 bucket notification -resource "aws_s3_bucket_notification" "cur_output" { - bucket = aws_s3_bucket.cur_output.id - - dynamic "lambda_function" { - for_each = local.s3_object_filters - content { - lambda_function_arn = aws_lambda_function.processing.arn - events = local.s3_notification_events - filter_suffix = lambda_function.value.suffix - } - } - - depends_on = [aws_lambda_permission.s3_invoke] -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/terraform.tfvars.example b/lambdas/functions/iroco2-client-side-scanner/tf/terraform.tfvars.example deleted file mode 100644 index d46fcc2..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/terraform.tfvars.example +++ /dev/null @@ -1,29 +0,0 @@ -# Example Terraform variables for IROCO2 Client Side Scanner -# Copy this file to terraform.tfvars and update with your actual values - -# Required variables -kms_key_arn = "arn:aws:kms:eu-west-3:992382519845:key/02094aa9-afab-4e3c-84b2-8082380c66eb" -layer_bucket_storage = "iroco-test-sylvain-scrapper" -layer_bucket_key = "layers/helper-scripts.zip" -aws_org_id = "o-yn7toj4ob7" -cur_output_bucket_name = "iroco-cur-bucket" -cur_function_s3_key = "functions/cur-scanner.zip" -cur_function_s3_bucket = "iroco-test-sylvain-scrapper" -iroco2_api_endpoint = "https://api.iroco2.example.com" -iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" -iroco2_api_key = "your-api-key-here" - -# Optional variables (uncomment to override defaults) -# lambda_function_name = "IROCO2-CUR-SCRAPPER" -# lambda_log_group_name = "/aws/lambda/IROCO2-CUR-SCRAPPER" -# lambda_timeout = 900 -# lambda_memory_size = 512 -# log_retention_days = 14 - -# Common tags -common_tags = { - Environment = "development" - Project = "IROCO2" - Owner = "data-team" - ManagedBy = "Terraform" -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/test-payload.json b/lambdas/functions/iroco2-client-side-scanner/tf/test-payload.json deleted file mode 100644 index 1c04e1b..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/test-payload.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Records": [ - { - "eventName": "ObjectCreated:Copy", - "s3": { - "bucket": { - "name": "iroco-cur-bucket" - }, - "object": { - "key": "iroco/iroco_CUR_report/20250901-20251001/20250918T091302Z/iroco_CUR_report-00001.csv.zip" - } - } - } - ] -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/variables.tf b/lambdas/functions/iroco2-client-side-scanner/tf/variables.tf deleted file mode 100644 index 6afa01a..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/variables.tf +++ /dev/null @@ -1,128 +0,0 @@ -variable "kms_key_arn" { - description = "The KMS key ARN to encrypt the bucket and logs" - type = string - validation { - condition = can(regex("^arn:aws:kms:", var.kms_key_arn)) - error_message = "The KMS key ARN must be a valid AWS KMS key ARN." - } -} - -variable "layer_bucket_storage" { - description = "The S3 bucket name for the lambda layer" - type = string -} - -variable "layer_bucket_key" { - description = "The S3 bucket key for the lambda layer" - type = string -} - -variable "aws_org_id" { - description = "The AWS Organization ID" - type = string - validation { - condition = can(regex("^o-[a-z0-9]{10,32}$", var.aws_org_id)) - error_message = "The AWS Organization ID must be in the format o-xxxxxxxxxx." - } -} - -variable "cur_output_bucket_name" { - description = "The S3 bucket name for the CUR output" - type = string - validation { - condition = can(regex("^[a-z0-9][a-z0-9-]*[a-z0-9]$", var.cur_output_bucket_name)) - error_message = "The bucket name must be a valid S3 bucket name." - } -} - -variable "cur_function_s3_key" { - description = "The S3 bucket key for the CUR lambda function" - type = string -} - -variable "cur_function_s3_bucket" { - description = "The S3 bucket name for the CUR lambda function" - type = string -} - -variable "iroco2_api_endpoint" { - description = "The IroCO2 API endpoint" - type = string - validation { - condition = can(regex("^https?://", var.iroco2_api_endpoint)) - error_message = "The API endpoint must be a valid HTTP or HTTPS URL." - } -} - -variable "iroco2_gateway_endpoint" { - description = "The IroCO2 Gateway endpoint" - type = string - validation { - condition = can(regex("^https?://", var.iroco2_gateway_endpoint)) - error_message = "The Gateway endpoint must be a valid HTTP or HTTPS URL." - } -} - -variable "iroco2_api_key" { - description = "The IroCO2 API token" - type = string - sensitive = true -} - -variable "lambda_log_group_name" { - description = "Log group name for lambda" - type = string - default = "/aws/lambda/IROCO2-CUR-SCRAPPER" -} - -variable "lambda_function_name" { - description = "Name of the Lambda function" - type = string - default = "IROCO2-CUR-SCRAPPER" -} - -variable "lambda_timeout" { - description = "Lambda function timeout in seconds" - type = number - default = 900 - validation { - condition = var.lambda_timeout >= 1 && var.lambda_timeout <= 900 - error_message = "Lambda timeout must be between 1 and 900 seconds." - } -} - -variable "lambda_memory_size" { - description = "Lambda function memory size in MB" - type = number - default = 512 - validation { - condition = var.lambda_memory_size >= 128 && var.lambda_memory_size <= 10240 - error_message = "Lambda memory size must be between 128 and 10240 MB." - } -} - -variable "log_retention_days" { - description = "CloudWatch log retention in days" - type = number - default = 14 - validation { - condition = contains([ - 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653 - ], var.log_retention_days) - error_message = "Log retention days must be a valid CloudWatch Logs retention value." - } -} - -variable "common_tags" { - description = "Common tags to apply to all resources" - type = map(string) - default = { - Project = "IROCO2" - } -} - -variable "aws_sdk_pandas_layer_arn" { - description = "ARN of the AWS SDK Pandas layer" - type = string - default = "arn:aws:lambda:eu-west-3:336392948345:layer:AWSSDKPandas-Python311:18" -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/versions.tf b/lambdas/functions/iroco2-client-side-scanner/tf/versions.tf deleted file mode 100644 index ddfcb0e..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/versions.tf +++ /dev/null @@ -1,10 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.0" - } - } -} From c906d29e4beb72ad01a52a0894e641a51f0c5bc8 Mon Sep 17 00:00:00 2001 From: sressier Date: Fri, 26 Sep 2025 10:59:05 +0200 Subject: [PATCH 5/9] use archive_file for lambda code --- .../iroco2-client-side-scanner/lambda.tf | 10 +- .../python/ec2_service.py | 99 --------------- .../python/iroco_service.py | 67 ---------- .../python/s3_repository_parser.py | 27 ---- .../python/s3_service.py | 116 ------------------ 5 files changed, 7 insertions(+), 312 deletions(-) delete mode 100644 lambdas/functions/iroco2-client-side-scanner/python/ec2_service.py delete mode 100644 lambdas/functions/iroco2-client-side-scanner/python/iroco_service.py delete mode 100644 lambdas/functions/iroco2-client-side-scanner/python/s3_repository_parser.py delete mode 100644 lambdas/functions/iroco2-client-side-scanner/python/s3_service.py diff --git a/lambdas/functions/iroco2-client-side-scanner/lambda.tf b/lambdas/functions/iroco2-client-side-scanner/lambda.tf index 7f80f1f..872dec0 100644 --- a/lambdas/functions/iroco2-client-side-scanner/lambda.tf +++ b/lambdas/functions/iroco2-client-side-scanner/lambda.tf @@ -4,6 +4,12 @@ data "archive_file" "layers" { output_path = "${path.module}/layers.zip" } +data "archive_file" "lambda_code" { + type = "zip" + source_dir = "${path.module}/package" + output_path = "${path.module}/lambda.zip" +} + # Lambda Layer for helper scripts resource "aws_lambda_layer_version" "helper_scripts" { layer_name = local.layer_name @@ -32,9 +38,7 @@ resource "aws_lambda_function" "processing" { var.aws_sdk_pandas_layer_arn ] - # Source code from S3 - s3_bucket = var.cur_function_s3_bucket - s3_key = var.cur_function_s3_key + filename = "lambda.zip" # Environment variables environment { diff --git a/lambdas/functions/iroco2-client-side-scanner/python/ec2_service.py b/lambdas/functions/iroco2-client-side-scanner/python/ec2_service.py deleted file mode 100644 index 7d58560..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/python/ec2_service.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -import pandas as pd -from io import StringIO, BytesIO -import isodate -from datetime import timedelta - -class CurProcessorEC2Service: - - cur_useful_column_keyname_matrix = { - 'service_code': { - 'zip': 'product/servicecode', - 'csv': 'product/servicecode', - 'parquet': 'product_servicecode' - }, - 'usage_type': { - 'zip': 'lineItem/UsageType', - 'csv': 'lineItem/UsageType', - 'parquet': 'line_item_usage_type' - }, - 'region_code': { - 'zip': 'product/regionCode', - 'csv': 'product/regionCode', - 'parquet': 'product_region_code' - }, - 'instance_type': { - 'zip': 'product/instanceType', - 'csv': 'product/instanceType', - 'parquet': 'product_instance_type' - } - } - - def creating_message_from_cur(self, cur, cur_file_type): - try: - useful_column = [values[cur_file_type] for values in self.cur_useful_column_keyname_matrix.values()] - match cur_file_type: - case 'zip': - cur_with_useful_colum = pd.read_csv(BytesIO(cur), usecols=useful_column, compression='zip', low_memory=False) - case 'csv': - cur_with_useful_colum = pd.read_csv(StringIO(cur.decode('utf-8')), usecols=useful_column, low_memory=False) - case 'parquet': - cur_with_useful_colum = pd.read_parquet(BytesIO(cur), columns=useful_column) - case _: - raise Exception(f'Unknown file type {cur_file_type}') - parsed_cur_by_service_ec2 = cur_with_useful_colum[ - cur_with_useful_colum[ - self.cur_useful_column_keyname_matrix['service_code'][cur_file_type] - ].isin(['AmazonEC2'])] - parsed_cur_by_service_ec2_and_box_usage = parsed_cur_by_service_ec2[ - parsed_cur_by_service_ec2[ - self.cur_useful_column_keyname_matrix['usage_type'][cur_file_type] - ].str.contains('BoxUsage', na=False)] - parsed_cur_ec2_grouped_by_instance_and_region = parsed_cur_by_service_ec2_and_box_usage.groupby( - [ - self.cur_useful_column_keyname_matrix['region_code'][cur_file_type], - self.cur_useful_column_keyname_matrix['instance_type'][cur_file_type] - ]).size().reset_index(name='numberOfLine') - - queue_messages = [] - for index, row in parsed_cur_ec2_grouped_by_instance_and_region.iterrows(): - queue_message = self.__create_parsed_data_message(row, cur_file_type) - queue_messages.append(queue_message) - - return queue_messages - except ValueError as e: - if 'Usecols do not match columns' in str(e): - print(f"ValueError: {e}") - return [] - else: - raise e - - def __create_parsed_data_message(self, row, cur_file_type): - aws_data_center = row[self.cur_useful_column_keyname_matrix['region_code'][cur_file_type]] - instance_type = row[self.cur_useful_column_keyname_matrix['instance_type'][cur_file_type]] - number_of_line = row['numberOfLine'] - duration = isodate.duration_isoformat(timedelta(hours=number_of_line)) - service_type_cur = "EC2" - - message = { - "ec2Type": instance_type, - "awsDataCenter": aws_data_center, - "durationOfServiceOperation": duration, - "serviceTypeCUR": service_type_cur - } - - return message diff --git a/lambdas/functions/iroco2-client-side-scanner/python/iroco_service.py b/lambdas/functions/iroco2-client-side-scanner/python/iroco_service.py deleted file mode 100644 index 987c350..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/python/iroco_service.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -import os -import requests - -class IrocoService: - - def __init__(self): - self.api_key = os.environ.get("IROCO2_API_KEY") - self.iroco_endpoint = os.environ.get("IROCO2_API_ENDPOINT") - self.gateway_endpoint = os.environ.get("IROCO2_GATEWAY_ENDPOINT") - - def __build_header(self): - """ - Build header - """ - return { - 'Authorization': f"Bearer {self.api_key}", - 'Content-Type': "application/json", - } - - def create_scan(self): - """ - Create a scan - """ - try: - response = requests.post(self.iroco_endpoint, headers=self.__build_header()) - response.raise_for_status() - except requests.exceptions.HTTPError as errh: - print("HTTP Error") - print(errh) - return None - return response.json() - - def send_payload(self, payload): - """ - Send a payload - """ - try: - send_report = requests.post(self.gateway_endpoint, json=payload, headers=self.__build_header()) - send_report.raise_for_status() - return send_report.status_code - except requests.exceptions.HTTPError as errh: - print("HTTP Error") - print(errh.args[0]) - return send_report.status_code - except requests.exceptions.ConnectionError as conerr: - print("Connection error") - print(conerr) - return send_report.status_code - except requests.exceptions.RequestException as errex: - print("Exception request") - print(errex) - return send_report.status_code diff --git a/lambdas/functions/iroco2-client-side-scanner/python/s3_repository_parser.py b/lambdas/functions/iroco2-client-side-scanner/python/s3_repository_parser.py deleted file mode 100644 index 30ae5c3..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/python/s3_repository_parser.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -import boto3 - -class S3Repository: - - def __init__(self): - self.s3_ressource = boto3.resource('s3') - - def read_file(self, bucket_name: str, key_to_read: str): - obj = self.s3_ressource.Object(bucket_name, key_to_read) - response = obj.get() - raw_content = response['Body'].read() - return raw_content diff --git a/lambdas/functions/iroco2-client-side-scanner/python/s3_service.py b/lambdas/functions/iroco2-client-side-scanner/python/s3_service.py deleted file mode 100644 index 48476d1..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/python/s3_service.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -import pandas as pd -import isodate - -from io import StringIO, BytesIO -from datetime import timedelta - - -class CurProcessorS3Service: - - cur_useful_column_keyname_matrix = { - 'service_code': { - 'zip': 'product/servicecode', - 'csv': 'product/servicecode', - 'parquet': 'product_servicecode' - }, - 'usage_type': { - 'zip': 'lineItem/UsageType', - 'csv': 'lineItem/UsageType', - 'parquet': 'line_item_usage_type' - }, - 'region_code': { - 'zip': 'product/regionCode', - 'csv': 'product/regionCode', - 'parquet': 'product_region_code' - }, - 'usage_amount': { - 'zip': 'lineItem/UsageAmount', - 'csv': 'lineItem/UsageAmount', - 'parquet': 'line_item_usage_amount' - } - } - - cur_file_type = None - - def creating_message_from_cur(self, cur, cur_file_type): - try: - self.cur_file_type = cur_file_type - queue_messages = [] - storage_queue_messages = self.__creating_message_from_cur_for_storage(cur) - queue_messages.extend(storage_queue_messages) - - return queue_messages - except ValueError as e: - if 'Usecols do not match columns' in str(e): - print(f"ValueError: {e}") - return [] - else: - raise e - - def __create_parsed_data_message(self, row): - aws_data_center = row[ - self.cur_useful_column_keyname_matrix['region_code'][self.cur_file_type] - ] - number_of_line = row['numberOfLines'] - total_usage_amount = row['totalUsageAmount'] - average_usage_amount = total_usage_amount / number_of_line - duration = isodate.duration_isoformat(timedelta(hours=number_of_line)) - service_type_cur = "S3" - - message = { - "storageInMo": average_usage_amount, - "awsDataCenter": aws_data_center, - "durationOfServiceOperation": duration, - "serviceTypeCUR": service_type_cur - } - - return message - - def __creating_message_from_cur_for_storage(self, cur): - useful_columns = [values[self.cur_file_type] for values in self.cur_useful_column_keyname_matrix.values()] - match self.cur_file_type: - case 'zip': - cur_with_useful_colum = pd.read_csv(BytesIO(cur), usecols=useful_columns, compression='zip', low_memory=False) - case 'csv': - cur_with_useful_colum = pd.read_csv(StringIO(cur.decode('utf-8')), usecols=useful_columns, low_memory=False) - case 'parquet': - cur_with_useful_colum = pd.read_parquet(BytesIO(cur), columns=useful_columns) - case _: - raise Exception(f'Unknown file type {self.cur_file_type}') - parsed_cur_by_service_s3 = ( - cur_with_useful_colum)[cur_with_useful_colum[ - self.cur_useful_column_keyname_matrix['service_code'][self.cur_file_type]] == 'AmazonS3' - ] - parsed_cur_by_service_s3_and_usage_type = parsed_cur_by_service_s3[ - parsed_cur_by_service_s3[ - self.cur_useful_column_keyname_matrix['usage_type'][self.cur_file_type] - ].str.contains('TimedStorage-ByteHrs', na=False)] - - parsed_cur_s3_grouped_by_instance_and_region = parsed_cur_by_service_s3_and_usage_type.groupby( - [self.cur_useful_column_keyname_matrix['region_code'][self.cur_file_type]] - ).agg( - totalUsageAmount=(self.cur_useful_column_keyname_matrix['usage_amount'][self.cur_file_type], 'sum'), - numberOfLines=(self.cur_useful_column_keyname_matrix['usage_amount'][self.cur_file_type], 'size') - ).reset_index() - - queue_messages = [] - for index, row in parsed_cur_s3_grouped_by_instance_and_region.iterrows(): - queue_message = self.__create_parsed_data_message(row) - queue_messages.append(queue_message) - - return queue_messages From a6aa54d5d0c295c39dac3603a5b9b4f0c17a5727 Mon Sep 17 00:00:00 2001 From: sressier Date: Fri, 26 Sep 2025 13:31:25 +0200 Subject: [PATCH 6/9] add CUR report, use modern expression --- .../iroco2-client-side-scanner/cur.tf | 34 +++++++++++++++++++ .../iroco2-client-side-scanner/s3.tf | 30 +++++++++++++++- 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 lambdas/functions/iroco2-client-side-scanner/cur.tf diff --git a/lambdas/functions/iroco2-client-side-scanner/cur.tf b/lambdas/functions/iroco2-client-side-scanner/cur.tf new file mode 100644 index 0000000..743a691 --- /dev/null +++ b/lambdas/functions/iroco2-client-side-scanner/cur.tf @@ -0,0 +1,34 @@ +resource "aws_bcmdataexports_export" "CUR" { + export { + name = "IROCO2-REPORT" + data_query { + query_statement = "SELECT identity_line_item_id, identity_time_interval, line_item_product_code,line_item_unblended_cost FROM COST_AND_USAGE_REPORT" + table_configurations = { + COST_AND_USAGE_REPORT = { + BILLING_VIEW_ARN = "arn:aws:billing::${data.aws_caller_identity.current.account_id}:billingview/primary" + TIME_GRANULARITY = "HOURLY", + INCLUDE_RESOURCES = "FALSE", + INCLUDE_MANUAL_DISCOUNT_COMPATIBILITY = "FALSE", + INCLUDE_SPLIT_COST_ALLOCATION_DATA = "FALSE", + } + } + } + destination_configurations { + s3_destination { + s3_bucket = aws_s3_bucket.cur_output.bucket + s3_prefix = "iroco" + s3_region = data.aws_region.current.region + s3_output_configurations { + overwrite = "CREATE_NEW_REPORT" + format = "TEXT_OR_CSV" + compression = "GZIP" + output_type = "CUSTOM" + } + } + } + + refresh_cadence { + frequency = "SYNCHRONOUS" + } + } +} diff --git a/lambdas/functions/iroco2-client-side-scanner/s3.tf b/lambdas/functions/iroco2-client-side-scanner/s3.tf index c3d79c9..a0cdb9c 100644 --- a/lambdas/functions/iroco2-client-side-scanner/s3.tf +++ b/lambdas/functions/iroco2-client-side-scanner/s3.tf @@ -45,7 +45,7 @@ resource "aws_s3_bucket_policy" "cur_output" { { Effect = "Allow" Principal = { - AWS = data.aws_caller_identity.current.account_id + AWS = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root" } Action = "s3:*" Resource = [ @@ -61,6 +61,34 @@ resource "aws_s3_bucket_policy" "cur_output" { } } }, + { + Effect = "Allow" + Principal = { + Service = [ + "bcm-data-exports.amazonaws.com", + "billingreports.amazonaws.com" + ] + } + Action = [ + "s3:GetBucketPolicy", + "s3:PutObject" + ] + Resource = [ + "arn:aws:s3:::${var.cur_output_bucket_name}", + "arn:aws:s3:::${var.cur_output_bucket_name}/*" + ] + Condition = { + StringEquals = { + "aws:SourceAccount" = data.aws_caller_identity.current.account_id, + }, + StringLike = { + "aws:SourceArn" = [ + "arn:aws:bcm-data-exports:us-east-1:${data.aws_caller_identity.current.account_id}:export/*", + "arn:aws:cur:us-east-1:${data.aws_caller_identity.current.account_id}:definition/*" + ] + } + } + }, { Effect = "Deny" Principal = { From 62d507d0f67a2cf6dd523031bcd2d6cec56723b1 Mon Sep 17 00:00:00 2001 From: sressier Date: Fri, 26 Sep 2025 13:42:40 +0200 Subject: [PATCH 7/9] update readme and variables --- .../iroco2-client-side-scanner/README.md | 18 +---------------- .../iroco2-client-side-scanner/variables.tf | 20 ------------------- 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/lambdas/functions/iroco2-client-side-scanner/README.md b/lambdas/functions/iroco2-client-side-scanner/README.md index 0f00a2e..5a18e66 100644 --- a/lambdas/functions/iroco2-client-side-scanner/README.md +++ b/lambdas/functions/iroco2-client-side-scanner/README.md @@ -69,16 +69,12 @@ The module provisions the following AWS resources: ```hcl module "iroco2_client_side_scanner" { - source = "./lambdas/functions/iroco2-client-side-scanner/tf" + source = "./lambdas/functions/iroco2-client-side-scanner/" # Required variables kms_key_arn = "arn:aws:kms:eu-west-3:123456789012:key/12345678-1234-1234-1234-123456789012" - layer_bucket_storage = "my-lambda-layers-bucket" - layer_bucket_key = "layers/helper-scripts.zip" aws_org_id = "o-1234567890" cur_output_bucket_name = "my-cur-output-bucket" - cur_function_s3_key = "functions/cur-scanner.zip" - cur_function_s3_bucket = "my-lambda-functions-bucket" iroco2_api_endpoint = "https://api.iroco2.example.com" iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" iroco2_api_key = "your-api-key-here" @@ -106,12 +102,8 @@ module "iroco2_client_side_scanner" { # Required variables kms_key_arn = data.aws_kms_key.main.arn - layer_bucket_storage = aws_s3_bucket.lambda_layers.bucket - layer_bucket_key = "layers/helper-scripts-v2.zip" aws_org_id = var.organization_id cur_output_bucket_name = "${var.environment}-cur-output-${random_id.bucket_suffix.hex}" - cur_function_s3_key = "functions/cur-scanner-${var.version}.zip" - cur_function_s3_bucket = aws_s3_bucket.lambda_functions.bucket iroco2_api_endpoint = var.iroco2_api_endpoint iroco2_gateway_endpoint = var.iroco2_gateway_endpoint iroco2_api_key = data.aws_secretsmanager_secret_version.iroco2_api_key.secret_string @@ -166,12 +158,8 @@ module "iroco2_client_side_scanner" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | kms_key_arn | The KMS key ARN to encrypt the bucket and logs | `string` | n/a | yes | -| layer_bucket_storage | The S3 bucket name for the lambda layer | `string` | n/a | yes | -| layer_bucket_key | The S3 bucket key for the lambda layer | `string` | n/a | yes | | aws_org_id | The AWS Organization ID | `string` | n/a | yes | | cur_output_bucket_name | The S3 bucket name for the CUR output | `string` | n/a | yes | -| cur_function_s3_key | The S3 bucket key for the CUR lambda function | `string` | n/a | yes | -| cur_function_s3_bucket | The S3 bucket name for the CUR lambda function | `string` | n/a | yes | | iroco2_api_endpoint | The IroCO2 API endpoint | `string` | n/a | yes | | iroco2_gateway_endpoint | The IroCO2 Gateway endpoint | `string` | n/a | yes | | iroco2_api_key | The IroCO2 API token | `string` | n/a | yes | @@ -409,12 +397,8 @@ You need to create these resources before deploying: ```hcl # Update these with your actual values kms_key_arn = "arn:aws:kms:eu-west-3:123456789012:key/abcd1234-..." -layer_bucket_storage = "my-lambda-layers-bucket" -layer_bucket_key = "layers/helper-scripts.zip" aws_org_id = "o-1234567890" cur_output_bucket_name = "iroco2-cur-output-unique-suffix" -cur_function_s3_key = "functions/cur-scanner.zip" -cur_function_s3_bucket = "my-lambda-functions-bucket" iroco2_api_endpoint = "https://api.iroco2.example.com" iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" iroco2_api_key = "your-secret-api-key" diff --git a/lambdas/functions/iroco2-client-side-scanner/variables.tf b/lambdas/functions/iroco2-client-side-scanner/variables.tf index 6afa01a..46d26c1 100644 --- a/lambdas/functions/iroco2-client-side-scanner/variables.tf +++ b/lambdas/functions/iroco2-client-side-scanner/variables.tf @@ -7,16 +7,6 @@ variable "kms_key_arn" { } } -variable "layer_bucket_storage" { - description = "The S3 bucket name for the lambda layer" - type = string -} - -variable "layer_bucket_key" { - description = "The S3 bucket key for the lambda layer" - type = string -} - variable "aws_org_id" { description = "The AWS Organization ID" type = string @@ -35,16 +25,6 @@ variable "cur_output_bucket_name" { } } -variable "cur_function_s3_key" { - description = "The S3 bucket key for the CUR lambda function" - type = string -} - -variable "cur_function_s3_bucket" { - description = "The S3 bucket name for the CUR lambda function" - type = string -} - variable "iroco2_api_endpoint" { description = "The IroCO2 API endpoint" type = string From d2bcc89d611d462071285e789e7743b29f106c6e Mon Sep 17 00:00:00 2001 From: sressier Date: Fri, 26 Sep 2025 14:14:49 +0200 Subject: [PATCH 8/9] update with working example --- .../iroco2-client-side-scanner/cur.tf | 2 +- .../examples/basic/main.tf | 48 +++++-------- .../examples/basic/variables.tf | 67 ------------------- .../iroco2-client-side-scanner/lambda.tf | 4 +- .../iroco2-client-side-scanner/main.tf | 5 -- .../iroco2-client-side-scanner/variables.tf | 6 ++ 6 files changed, 25 insertions(+), 107 deletions(-) delete mode 100644 lambdas/functions/iroco2-client-side-scanner/examples/basic/variables.tf diff --git a/lambdas/functions/iroco2-client-side-scanner/cur.tf b/lambdas/functions/iroco2-client-side-scanner/cur.tf index 743a691..8ad0793 100644 --- a/lambdas/functions/iroco2-client-side-scanner/cur.tf +++ b/lambdas/functions/iroco2-client-side-scanner/cur.tf @@ -1,6 +1,6 @@ resource "aws_bcmdataexports_export" "CUR" { export { - name = "IROCO2-REPORT" + name = var.bcm_data_export_name data_query { query_statement = "SELECT identity_line_item_id, identity_time_interval, line_item_product_code,line_item_unblended_cost FROM COST_AND_USAGE_REPORT" table_configurations = { diff --git a/lambdas/functions/iroco2-client-side-scanner/examples/basic/main.tf b/lambdas/functions/iroco2-client-side-scanner/examples/basic/main.tf index aa85a80..0849f6d 100644 --- a/lambdas/functions/iroco2-client-side-scanner/examples/basic/main.tf +++ b/lambdas/functions/iroco2-client-side-scanner/examples/basic/main.tf @@ -14,47 +14,31 @@ provider "aws" { region = var.aws_region } -# Data sources for existing resources -data "aws_kms_key" "existing" { - key_id = var.kms_key_id -} -# Example S3 buckets (you may want to use existing ones) -resource "aws_s3_bucket" "lambda_artifacts" { - bucket = "${var.project_name}-lambda-artifacts-${random_id.suffix.hex}" -} -resource "random_id" "suffix" { - byte_length = 4 -} # Use the IROCO2 Client Side Scanner module module "iroco2_scanner" { - source = "../" + source = "../../." # Required variables - kms_key_arn = data.aws_kms_key.existing.arn - layer_bucket_storage = aws_s3_bucket.lambda_artifacts.bucket - layer_bucket_key = aws_s3_object.lambda_layer.key - aws_org_id = var.aws_org_id - cur_output_bucket_name = "${var.project_name}-cur-output-${random_id.suffix.hex}" - cur_function_s3_key = aws_s3_object.lambda_function.key - cur_function_s3_bucket = aws_s3_bucket.lambda_artifacts.bucket - iroco2_api_endpoint = var.iroco2_api_endpoint - iroco2_gateway_endpoint = var.iroco2_gateway_endpoint - iroco2_api_key = var.iroco2_api_key - - # Optional customization - lambda_function_name = "${var.project_name}-cur-scanner" - lambda_timeout = var.lambda_timeout - lambda_memory_size = var.lambda_memory_size - + kms_key_arn = "arn:aws:kms:eu-west-3:992382519845:key/f4ab6c91-0a7d-4a32-8c11-8fdd4380888d" + aws_org_id = "o-yn7toj4ob7" + cur_output_bucket_name = "iroco-cur-bucket-toto" + iroco2_api_endpoint = "https://api.iroco2.example.com" + iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" + iroco2_api_key = "your-api-key-here" + bcm_data_export_name = "IROCO2-REPORT-TEST" + lambda_function_name = "test_deploy_iroco2" + lambda_log_group_name = "test-deploy-iroco2" +# Common tags common_tags = { - Environment = var.environment - Project = var.project_name - Owner = var.owner - Repository = "https://github.com/ippontech/iroco2-lambdas" + Environment = "development" + Project = "IROCO2" + Owner = "data-team" + ManagedBy = "Terraform" } + } # Outputs diff --git a/lambdas/functions/iroco2-client-side-scanner/examples/basic/variables.tf b/lambdas/functions/iroco2-client-side-scanner/examples/basic/variables.tf deleted file mode 100644 index a2782a8..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/examples/basic/variables.tf +++ /dev/null @@ -1,67 +0,0 @@ -# Variables for the basic example - -variable "aws_region" { - description = "AWS region to deploy resources" - type = string - default = "eu-west-3" -} - -variable "project_name" { - description = "Name of the project (used for resource naming)" - type = string - default = "iroco2" -} - -variable "environment" { - description = "Environment name (dev, staging, prod)" - type = string - default = "dev" -} - -variable "owner" { - description = "Owner of the resources" - type = string - default = "data-team" -} - -variable "kms_key_id" { - description = "KMS key ID or alias for encryption (must support ENCRYPT_DECRYPT)" - type = string - # Example: "alias/aws/s3" or "12345678-1234-1234-1234-123456789012" -} - -variable "aws_org_id" { - description = "AWS Organization ID for access control" - type = string - # Example: "o-1234567890" -} - -variable "iroco2_api_endpoint" { - description = "IROCO2 API endpoint URL" - type = string - # Example: "https://api.iroco2.example.com" -} - -variable "iroco2_gateway_endpoint" { - description = "IROCO2 Gateway endpoint URL" - type = string - # Example: "https://gateway.iroco2.example.com" -} - -variable "iroco2_api_key" { - description = "IROCO2 API authentication key" - type = string - sensitive = true -} - -variable "lambda_timeout" { - description = "Lambda function timeout in seconds" - type = number - default = 900 -} - -variable "lambda_memory_size" { - description = "Lambda function memory size in MB" - type = number - default = 512 -} diff --git a/lambdas/functions/iroco2-client-side-scanner/lambda.tf b/lambdas/functions/iroco2-client-side-scanner/lambda.tf index 872dec0..2eb88c5 100644 --- a/lambdas/functions/iroco2-client-side-scanner/lambda.tf +++ b/lambdas/functions/iroco2-client-side-scanner/lambda.tf @@ -16,7 +16,7 @@ resource "aws_lambda_layer_version" "helper_scripts" { description = "Scripts for IROCO2 helpers" compatible_runtimes = ["python3.9", "python3.10", "python3.11"] - filename = "layers.zip" + filename = "${path.module}/layers.zip" } @@ -38,7 +38,7 @@ resource "aws_lambda_function" "processing" { var.aws_sdk_pandas_layer_arn ] - filename = "lambda.zip" + filename = "${path.module}/lambda.zip" # Environment variables environment { diff --git a/lambdas/functions/iroco2-client-side-scanner/main.tf b/lambdas/functions/iroco2-client-side-scanner/main.tf index e84dfc1..e69de29 100644 --- a/lambdas/functions/iroco2-client-side-scanner/main.tf +++ b/lambdas/functions/iroco2-client-side-scanner/main.tf @@ -1,5 +0,0 @@ -# Main Terraform configuration for IROCO2 Client Side Scanner -# This module creates all necessary AWS resources for the CUR scanner Lambda function - -# This file serves as the main entry point and can include any additional -# configuration or data sources that don't fit in the specific resource files diff --git a/lambdas/functions/iroco2-client-side-scanner/variables.tf b/lambdas/functions/iroco2-client-side-scanner/variables.tf index 46d26c1..0971e62 100644 --- a/lambdas/functions/iroco2-client-side-scanner/variables.tf +++ b/lambdas/functions/iroco2-client-side-scanner/variables.tf @@ -106,3 +106,9 @@ variable "aws_sdk_pandas_layer_arn" { type = string default = "arn:aws:lambda:eu-west-3:336392948345:layer:AWSSDKPandas-Python311:18" } + +variable "bcm_data_export_name" { + description = "Name for the CUR data export in the console" + type = string + default = "IROCO2-REPORT" +} From c5b54fd6c16a0941ed736104bb185ecec0c9b5a1 Mon Sep 17 00:00:00 2001 From: sressier Date: Fri, 26 Sep 2025 14:19:00 +0200 Subject: [PATCH 9/9] update README --- lambdas/functions/iroco2-client-side-scanner/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lambdas/functions/iroco2-client-side-scanner/README.md b/lambdas/functions/iroco2-client-side-scanner/README.md index 5a18e66..37c4bda 100644 --- a/lambdas/functions/iroco2-client-side-scanner/README.md +++ b/lambdas/functions/iroco2-client-side-scanner/README.md @@ -159,7 +159,8 @@ module "iroco2_client_side_scanner" { |------|-------------|------|---------|:--------:| | kms_key_arn | The KMS key ARN to encrypt the bucket and logs | `string` | n/a | yes | | aws_org_id | The AWS Organization ID | `string` | n/a | yes | -| cur_output_bucket_name | The S3 bucket name for the CUR output | `string` | n/a | yes | +| cur_output_bucket_name | Name for the CUR data export in the console | `string` | n/a | yes | +| bcm_data_export_name | The S3 bucket name for the CUR output | `string` | IROCO2-REPORT | no | | iroco2_api_endpoint | The IroCO2 API endpoint | `string` | n/a | yes | | iroco2_gateway_endpoint | The IroCO2 Gateway endpoint | `string` | n/a | yes | | iroco2_api_key | The IroCO2 API token | `string` | n/a | yes | @@ -399,6 +400,7 @@ You need to create these resources before deploying: kms_key_arn = "arn:aws:kms:eu-west-3:123456789012:key/abcd1234-..." aws_org_id = "o-1234567890" cur_output_bucket_name = "iroco2-cur-output-unique-suffix" +bcm_data_export_name = "IROCO2-REPORT" iroco2_api_endpoint = "https://api.iroco2.example.com" iroco2_gateway_endpoint = "https://gateway.iroco2.example.com" iroco2_api_key = "your-secret-api-key"