diff --git a/.github/workflows/package-analyzer-lambda.yml b/.github/workflows/package-analyzer-lambda.yml new file mode 100644 index 0000000..2959712 --- /dev/null +++ b/.github/workflows/package-analyzer-lambda.yml @@ -0,0 +1,49 @@ +name: Package CUR Analyzer Lambda + +on: + push: + tags: + - 'analyzer-v*' + +permissions: + contents: write + +jobs: + build-and-release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up environment + id: setup_env + run: | + FUNC_DIR='lambdas/functions/iroco2-cur-analyzer' + OUT_DIR=${FUNC_DIR}/dist + echo "FUNC_DIR=${FUNC_DIR}" >> $GITHUB_ENV + echo "HANDLER_DIR=${FUNC_DIR}/src" >> $GITHUB_ENV + echo "OUT_DIR=${OUT_DIR}" >> $GITHUB_ENV + mkdir -p ${OUT_DIR} + + - name: Build layer + run: | + cd ${{ env.FUNC_DIR }} + bash scripts/python-layers.sh "${{ github.workspace }}/${{ env.OUT_DIR }}/analyzer-layers.zip" + shell: bash + + - name: Create handler.zip + run: | + cd "${{ env.HANDLER_DIR }}" + zip -r "../dist/analyzer-handler.zip" . -x "*/.*" + shell: bash + + - name: Create Release and Upload Assets + uses: softprops/action-gh-release@v1 + with: + append_body: true + body: | + This release refer only to the CUR analyzer lambda. + files: | + ${{ env.OUT_DIR }}/analyzer-layers.zip + ${{ env.OUT_DIR }}/analyzer-handler.zip + diff --git a/.github/workflows/package-authorizer-lambda.yml b/.github/workflows/package-authorizer-lambda.yml new file mode 100644 index 0000000..d963271 --- /dev/null +++ b/.github/workflows/package-authorizer-lambda.yml @@ -0,0 +1,39 @@ +name: Package Scanner Authorizer Lambda + +on: + push: + tags: + - 'authorizer-v*' + +permissions: + contents: write + +jobs: + build-and-release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up environment + id: setup_env + run: | + FUNC_DIR='lambdas/functions/iroco2-authorizer' + echo "FUNC_DIR=${FUNC_DIR}" >> $GITHUB_ENV + echo "HANDLER_DIR=${FUNC_DIR}/src" >> $GITHUB_ENV + + - name: Build layer + run: | + cd ${{ env.HANDLER_DIR }} + ../scripts/build.sh + + - name: Create Release and Upload Assets + uses: softprops/action-gh-release@v1 + with: + append_body: true + body: | + This release refer only to the scanner authorizer lambda. + files: | + ${{ env.HANDLER_DIR }}/bootstrap + + \ No newline at end of file diff --git a/.github/workflows/package-scanner-lambda.yml b/.github/workflows/package-scanner-lambda.yml new file mode 100644 index 0000000..f7590af --- /dev/null +++ b/.github/workflows/package-scanner-lambda.yml @@ -0,0 +1,53 @@ +name: Package Scanner Lambda + +on: + push: + tags: + - 'scanner-v*' + +permissions: + contents: write + +jobs: + build-and-release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up environment + id: setup_env + run: | + FUNC_DIR='lambdas/functions/iroco2-client-side-scanner' + OUT_DIR=${FUNC_DIR}/dist + echo "FUNC_DIR=${FUNC_DIR}" >> $GITHUB_ENV + echo "LAYERS_DIR=${FUNC_DIR}/layers" >> $GITHUB_ENV + echo "HANDLER_DIR=${FUNC_DIR}/package" >> $GITHUB_ENV + echo "OUT_DIR=${OUT_DIR}" >> $GITHUB_ENV + mkdir -p ${OUT_DIR} + + - name: Create layers.zip + run: | + cd "${{ env.LAYERS_DIR }}" + zip -r "../dist/scanner-layers.zip" . -x "*/.*" + shell: bash + + - name: Create handler.zip + run: | + cd "${{ env.HANDLER_DIR }}" + zip -r "../dist/scanner-handler.zip" . -x "*/.*" + shell: bash + + - name: Prepare config for release + run: cp "${{ env.FUNC_DIR }}/lambda.yaml" "${{ env.OUT_DIR }}/scanner-lambda.yaml" + + - name: Create Release and Upload Assets + uses: softprops/action-gh-release@v1 + with: + append_body: true + body: | + This release refer only to the client side scanner lambda. + files: | + ${{ env.OUT_DIR }}/scanner-layers.zip + ${{ env.OUT_DIR }}/scanner-handler.zip + ${{ env.OUT_DIR }}/scanner-lambda.yaml diff --git a/lambdas/functions/iroco2-authorizer/src/build.sh b/lambdas/functions/iroco2-authorizer/scripts/build.sh similarity index 100% rename from lambdas/functions/iroco2-authorizer/src/build.sh rename to lambdas/functions/iroco2-authorizer/scripts/build.sh diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/00-backend.tf b/lambdas/functions/iroco2-client-side-scanner/tf/00-backend.tf deleted file mode 100644 index 8fba535..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/00-backend.tf +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -terraform { - backend "s3" { - key = "lambda-scrapper-client/eu-west-3/terraform.tfstate" - } -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/00-datasources.tf b/lambdas/functions/iroco2-client-side-scanner/tf/00-datasources.tf deleted file mode 100644 index a46f343..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/00-datasources.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -data "aws_region" "current" {} - -data "aws_caller_identity" "current" {} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/00-outputs.tf b/lambdas/functions/iroco2-client-side-scanner/tf/00-outputs.tf deleted file mode 100644 index cec3e52..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/00-outputs.tf +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -output "scrapper_bucket_name" { - value = aws_s3_bucket.scrapper_bucket.bucket -} - -output "layers_bucket_key" { - value = aws_s3_object.scrapper_layers.key -} - -output "handler_bucket_key" { - value = aws_s3_object.scrapper_handler.key -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/00-providers.tf b/lambdas/functions/iroco2-client-side-scanner/tf/00-providers.tf deleted file mode 100644 index 6b55ef6..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/00-providers.tf +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -provider "aws" { - default_tags { - tags = { - namespace = var.namespace - project_type = var.project_type - project = var.project_name - environment = var.environment - } - } -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/00-variables.tf b/lambdas/functions/iroco2-client-side-scanner/tf/00-variables.tf deleted file mode 100644 index e08a77e..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/00-variables.tf +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -variable "namespace" { - type = string - description = "The namespace in which the project is." - default = "iroco2" -} - -variable "environment" { - type = string - description = "The name of the environment we are deploying to" -} - -variable "project_name" { - type = string - description = "Project's name" - default = "scrapper" -} - -variable "project_type" { - type = string - description = "The type of project." - default = "lambda" -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/00-versions.tf b/lambdas/functions/iroco2-client-side-scanner/tf/00-versions.tf deleted file mode 100644 index 21869d8..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/00-versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -terraform { - required_version = ">= 1.0.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = "5.68.0" - } - archive = { - source = "hashicorp/archive" - version = "2.6.0" - } - } -} diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/10-s3.tf b/lambdas/functions/iroco2-client-side-scanner/tf/10-s3.tf deleted file mode 100644 index bb865fd..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/10-s3.tf +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -locals { - layers_key = "${var.namespace}/${var.project_name}/layers.zip" - layers_path = "${path.module}/../layers" - layers_zip_path = "${local.layers_path}/layers.zip" - layers = fileset(local.layers_path, "**/*.py") - handler_key = "${var.namespace}/${var.project_name}/handler.zip" - handler_path = "${path.module}/../package" - handler_zip_path = "${local.handler_path}/cur_scrapper.zip" -} - -data "archive_file" "layers" { - type = "zip" - output_path = local.layers_zip_path - - dynamic "source" { - for_each = local.layers - content { - filename = source.value - content = file("${local.layers_path}/${source.value}") - } - } -} - -data "archive_file" "handler" { - type = "zip" - source_dir = local.handler_path - output_path = local.handler_zip_path - - excludes = ["${basename(local.handler_zip_path)}"] -} - -resource "aws_s3_bucket" "scrapper_bucket" { - bucket = "${var.namespace}-${var.project_type}-${var.project_name}-${var.environment}" -} - -resource "aws_s3_bucket_ownership_controls" "scrapper_bucket" { - bucket = aws_s3_bucket.scrapper_bucket.id - rule { - object_ownership = "BucketOwnerPreferred" - } -} - -resource "aws_s3_bucket_acl" "scrapper_bucket" { - depends_on = [aws_s3_bucket_ownership_controls.scrapper_bucket] - bucket = aws_s3_bucket.scrapper_bucket.id - acl = "private" -} - -data "aws_iam_policy_document" "scrapper_bucket_policy" { - statement { - sid = "PublicReadForLambdaFiles" - principals { - type = "*" - identifiers = ["*"] - } - actions = [ - "s3:GetObject", - ] - resources = [ - "${aws_s3_bucket.scrapper_bucket.arn}/${local.layers_key}", - "${aws_s3_bucket.scrapper_bucket.arn}/${local.handler_key}" - ] - } - - statement { - sid = "AllowListBucket" - principals { - type = "*" - identifiers = ["*"] - } - actions = [ - "s3:ListBucket", - ] - resources = [ - aws_s3_bucket.scrapper_bucket.arn - ] - } -} - -resource "aws_s3_bucket_policy" "scrapper_bucket" { - bucket = aws_s3_bucket.scrapper_bucket.id - policy = data.aws_iam_policy_document.scrapper_bucket_policy.json -} - -resource "aws_s3_object" "scrapper_layers" { - bucket = aws_s3_bucket.scrapper_bucket.bucket - key = local.layers_key - source = data.archive_file.layers.output_path - etag = filemd5(data.archive_file.layers.output_path) - acl = "public-read" -} - -resource "aws_s3_object" "scrapper_handler" { - bucket = aws_s3_bucket.scrapper_bucket.bucket - key = local.handler_key - source = data.archive_file.handler.output_path - etag = filemd5(data.archive_file.handler.output_path) - acl = "public-read" -} - -resource "aws_s3_bucket_request_payment_configuration" "scrapper_bucket" { - bucket = aws_s3_bucket.scrapper_bucket.id - payer = "BucketOwner" -} - -resource "aws_s3_bucket_public_access_block" "scrapper_bucket" { - bucket = aws_s3_bucket.scrapper_bucket.id - - block_public_acls = false - block_public_policy = false - ignore_public_acls = false - restrict_public_buckets = false -} - diff --git a/lambdas/functions/iroco2-client-side-scanner/tf/20-cfn-s3.tf b/lambdas/functions/iroco2-client-side-scanner/tf/20-cfn-s3.tf deleted file mode 100644 index 057a5c9..0000000 --- a/lambdas/functions/iroco2-client-side-scanner/tf/20-cfn-s3.tf +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2025 Ippon Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -resource "aws_s3_bucket" "cfn_bucket" { - bucket = "${var.namespace}-cfn-templates-${var.environment}" -} - -resource "aws_s3_bucket_ownership_controls" "cfn_bucket" { - bucket = aws_s3_bucket.cfn_bucket.id - rule { - object_ownership = "BucketOwnerPreferred" - } -} - -resource "aws_s3_bucket_acl" "cfn_bucket" { - depends_on = [aws_s3_bucket_ownership_controls.cfn_bucket] - bucket = aws_s3_bucket.cfn_bucket.id - acl = "private" -} - -resource "aws_s3_bucket_public_access_block" "cfn_bucket" { - bucket = aws_s3_bucket.cfn_bucket.id - - block_public_acls = false - block_public_policy = false - ignore_public_acls = false - restrict_public_buckets = false -} - -data "aws_iam_policy_document" "cfn_bucket_policy" { - statement { - principals { - type = "*" - identifiers = ["*"] - } - actions = [ - "s3:GetObject", - ] - resources = [ - "${aws_s3_bucket.cfn_bucket.arn}/*", - ] - } -} - -resource "aws_s3_bucket_policy" "cfn_bucket" { - bucket = aws_s3_bucket.cfn_bucket.id - policy = data.aws_iam_policy_document.cfn_bucket_policy.json -} - -resource "aws_s3_bucket_request_payment_configuration" "cfn_bucket" { - bucket = aws_s3_bucket.cfn_bucket.id - payer = "BucketOwner" -} - -resource "aws_s3_object" "lambda_yaml" { - bucket = aws_s3_bucket.cfn_bucket.bucket - key = "lambda.yaml" - source = "${path.module}/../lambda.yaml" - etag = filemd5("${path.module}/../lambda.yaml") - acl = "public-read" -} - -output "lambda_yaml_url" { - description = "URL of the publicly accessible lambda.yaml file" - value = "https://${aws_s3_bucket.cfn_bucket.bucket}.s3.${data.aws_region.current.name}.amazonaws.com/lambda.yaml" -} diff --git a/lambdas/functions/iroco2-cur-analyzer/requirements.txt b/lambdas/functions/iroco2-cur-analyzer/requirements.txt new file mode 100644 index 0000000..e1e8d39 --- /dev/null +++ b/lambdas/functions/iroco2-cur-analyzer/requirements.txt @@ -0,0 +1,6 @@ +boto3==1.35.72 +isodate==0.7.2 +pandas==2.1.4 +numpy==1.26.4 +pyarrow==14.0.1 +# Using compatible versions of pandas, numpy, and pyarrow to avoid dependency conflicts \ No newline at end of file diff --git a/lambdas/functions/iroco2-cur-analyzer/scripts/python-layers.sh b/lambdas/functions/iroco2-cur-analyzer/scripts/python-layers.sh index c761b4d..b365e9b 100755 --- a/lambdas/functions/iroco2-cur-analyzer/scripts/python-layers.sh +++ b/lambdas/functions/iroco2-cur-analyzer/scripts/python-layers.sh @@ -1,24 +1,33 @@ #!/usr/bin/env sh -# This script is used within GitLab CI to create the proper zip files for the layers. -# This should not be ran locally when using localstack since localstack free can't use lambda layers. +# This script is used within GitLab CI to create the proper zip file for the lambda layer. set -o errexit +set -o nounset -echo "Creating zip file for lambda layers" -mkdir -p ./layers +if [ -z "$1" ]; then + echo "Error: Output zip file path is required as the first argument." >&2 + exit 1 +fi -SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +OUTPUT_ZIP_PATH=$1 -# Create a single consolidated layer with all dependencies -echo "Creating python directory for the layer" -mkdir -p ./python +echo "Creating zip file for lambda layers at ${OUTPUT_ZIP_PATH}" -echo "Installing all dependencies" -pip install --platform manylinux2014_x86_64 --only-binary=:all: --target ./python -r requirements.txt +# Create a temporary directory for building the layer to avoid polluting the source tree +BUILD_DIR=$(mktemp -d) -echo "Compressing dependencies into a single layer" -python -m zipfile -c "./layers/dependencies.zip" "./python/" +# Create the python directory structure required by Lambda +mkdir -p "${BUILD_DIR}/python" -echo "Cleaning up" -rm -Rf "./python" +echo "Installing all dependencies into ${BUILD_DIR}/python" +pip install --platform manylinux2014_x86_64 --only-binary=:all: --target "${BUILD_DIR}/python" -r requirements.txt + +echo "Compressing dependencies into ${OUTPUT_ZIP_PATH}" +cd "${BUILD_DIR}/python" +zip -r "${OUTPUT_ZIP_PATH}" . + +# Clean up the temporary directory +rm -rf "${BUILD_DIR}" + +echo "Layer zip created successfully." diff --git a/lambdas/functions/iroco2-cur-analyzer/tf/10-lambda.tf b/lambdas/functions/iroco2-cur-analyzer/tf/10-lambda.tf index c4cfb0f..194d7a3 100644 --- a/lambdas/functions/iroco2-cur-analyzer/tf/10-lambda.tf +++ b/lambdas/functions/iroco2-cur-analyzer/tf/10-lambda.tf @@ -27,6 +27,7 @@ locals { } } +# TODO: Retrieve the zip file from the GitHub release instead of zipping it here data "archive_file" "cur_processor" { type = "zip"