Phase 4: toolchain cleanup + backend driver swap

source/app/alembic/alembic_utils.py

@@ -1,43 +1,54 @@
 from alembic import op
-from sqlalchemy import engine_from_config
-from sqlalchemy.engine import reflection
-
-
-from sqlalchemy import text
+from sqlalchemy import inspect, text
 
 
 def _table_has_column(table, column):
-    config = op.get_context().config
-    engine = engine_from_config(
-        config.get_section(config.config_ini_section), prefix='sqlalchemy.')
-    connection = engine.connect()
-    try:
-        result = connection.execute(text(f"SELECT * FROM \"{table}\" LIMIT 1"))
-        columns = result.keys()
-    except Exception:
-        return False
-    finally:
-        connection.close()
-
-    has_column = column in columns
-    return has_column
+    """Check whether *column* exists in *table* using the active migration connection.
+
+    Reuses op.get_bind() so the check runs inside the same database transaction
+    that the migration is already holding.  Opening a separate connection (the
+    previous approach) caused a self-deadlock on PostgreSQL with psycopg3 /
+    SQLAlchemy 2.0: the migration connection held an ACCESS EXCLUSIVE lock on
+    the table (from a preceding ALTER TABLE) while the new connection blocked
+    waiting for that same lock to be released – a wait that could never resolve
+    because the migration thread was synchronously waiting for _table_has_column
+    to return.
+
+    The previous implementation queried ``SELECT * FROM "<table>" LIMIT 1``.
+    On a completely fresh database (no prior ``db.create_all()``), that query
+    raises ``UndefinedTable`` which—with psycopg3 inside an active
+    transaction—permanently aborts the transaction.  The bare
+    ``except Exception: return False`` clause suppressed the Python exception
+    but did not issue ``ROLLBACK`` or ``ROLLBACK TO SAVEPOINT``, so every
+    subsequent DDL statement in the same alembic run failed with
+    ``InFailedSqlTransaction``.  This is the root cause of the CI health-check
+    failure on a fresh DB with psycopg3.
+
+    Fix: query ``information_schema.columns`` instead.  That catalog view always
+    exists and never raises, so the transaction is never put into an aborted
+    state regardless of whether *table* or *column* exists.
+    """
+    bind = op.get_bind()
+    result = bind.execute(
+        text(
+            "SELECT 1 FROM information_schema.columns"
+            " WHERE table_schema = 'public'"
+            " AND table_name = :tbl AND column_name = :col"
+        ),
+        {"tbl": table, "col": column},
+    )
+    return result.fetchone() is not None
 
 
 def _has_table(table_name):
-    config = op.get_context().config
-    engine = engine_from_config(
-        config.get_section(config.config_ini_section), prefix="sqlalchemy."
-    )
-    inspector = reflection.Inspector.from_engine(engine)
+    bind = op.get_bind()
+    inspector = inspect(bind)
     tables = inspector.get_table_names()
     return table_name in tables
 
 
 def index_exists(table_name, index_name):
-    config = op.get_context().config
-    engine = engine_from_config(
-        config.get_section(config.config_ini_section), prefix="sqlalchemy."
-    )
-    inspector = reflection.Inspector.from_engine(engine)
+    bind = op.get_bind()
+    inspector = inspect(bind)
     indexes = inspector.get_indexes(table_name)
     return any(index['name'] == index_name for index in indexes)

source/app/blueprints/graphql/cases.py

@@ -29,6 +29,7 @@
 
 from app.blueprints.access_controls import ac_current_user_has_customer_access
 from app.models.cases import Cases
+from app.db import db
 from app.models.authorization import Permissions
 from app.models.authorization import CaseAccessLevel
 
@@ -70,7 +71,7 @@ def resolve_iocs(root, info, ioc_id=None, ioc_uuid=None, ioc_value=None, ioc_typ
     @staticmethod
     def resolve_case(root, info, case_id):
         permissions_check_current_user_has_some_case_access(case_id, [CaseAccessLevel.full_access])
-        return Cases.query.get(case_id)
+        return db.session.get(Cases, case_id)
 
 
 class CaseConnection(Connection):

source/app/blueprints/rest/case/case_notes_routes.py

@@ -328,6 +328,7 @@ def case_directory_add(caseid):
 @ac_requires_case_identifier(CaseAccessLevel.full_access)
 @ac_api_requires()
 def case_directory_update(dir_id, caseid):
+    dir_id = int(dir_id)
     try:
 
         directory = _get_directory_for_case(dir_id, caseid)
@@ -363,6 +364,7 @@ def case_directory_update(dir_id, caseid):
 @ac_requires_case_identifier(CaseAccessLevel.full_access)
 @ac_api_requires()
 def case_directory_delete(dir_id, caseid):
+    dir_id = int(dir_id)
     try:
 
         directory = _get_directory_for_case(dir_id, caseid)

source/app/blueprints/rest/manage/manage_templates_routes.py

@@ -140,6 +140,7 @@ def add_template():
 @manage_templates_rest_blueprint.route('/manage/templates/download/<report_id>', methods=['GET'])
 @ac_api_requires(Permissions.server_administrator)
 def download_template(report_id):
+    report_id = int(report_id)
     if report_id != 0:
         report_template = CaseTemplateReport.query.filter(CaseTemplateReport.id == report_id).first()
 
@@ -154,6 +155,7 @@ def download_template(report_id):
 @manage_templates_rest_blueprint.route('/manage/templates/delete/<report_id>', methods=['POST'])
 @ac_api_requires(Permissions.server_administrator)
 def delete_template(report_id):
+    report_id = int(report_id)
     error = None
 
     report_template = CaseTemplateReport.query.filter(CaseTemplateReport.id == report_id).first()

source/app/blueprints/rest/v2/alerts.py

@@ -16,6 +16,8 @@
 #  along with this program; if not, write to the Free Software Foundation,
 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
+import sqlalchemy.exc
+
 from flask import Blueprint
 from flask import session
 from flask import request
@@ -46,6 +48,7 @@
 from app.business.alerts import alerts_get_related
 from app.models.errors import BusinessProcessingError
 from app.models.errors import ObjectNotFoundError
+from app.db import db
 
 
 class AlertsOperations:
@@ -277,6 +280,10 @@ def delete(self, identifier):
         except ObjectNotFoundError:
             return response_api_not_found()
 
+        except sqlalchemy.exc.OperationalError:
+            db.session.rollback()
+            return response_api_error("Alert deletion failed due to database error", status=500)
+
 
 alerts_blueprint = Blueprint('alerts_rest_v2', __name__, url_prefix='/alerts')
 alerts_blueprint.register_blueprint(alerts_comments_blueprint)

source/app/blueprints/rest/v2/cases.py

@@ -71,7 +71,7 @@ def search(self):
 
         case_ids_str = request.args.get('case_ids', None, type=parse_comma_separated_identifiers)
 
-        case_customer_id = request.args.get('case_customer_id', None, type=str)
+        case_customer_id = request.args.get('case_customer_id', None, type=int)
         case_name = request.args.get('case_name', None, type=str)
         case_description = request.args.get('case_description', None, type=str)
         case_classification_id = request.args.get(
@@ -178,7 +178,7 @@ def filter(self) -> Response:
             except ValueError:
                 return response_api_error('Invalid case id')
 
-        case_customer_id = request.args.get('case_customer_id', None, type=str)
+        case_customer_id = request.args.get('case_customer_id', None, type=int)
         case_name = request.args.get('case_name', None, type=str)
         case_description = request.args.get('case_description', None, type=str)
         case_classification_id = request.args.get('case_classification_id', None, type=int)

source/app/configuration.py

@@ -161,9 +161,9 @@ def _load_file_deprecated(self, section, option):
 
 
 # Build of SQLAlchemy connectors. One is admin and the other is only for iris. Admin is needed to create new DB
-SQLALCHEMY_BASE_URI = f'postgresql+psycopg2://{PG_ACCOUNT_}:{PG_PASSWD_}@{PG_SERVER_}:{PG_PORT_}/'
+SQLALCHEMY_BASE_URI = f'postgresql+psycopg://{PG_ACCOUNT_}:{PG_PASSWD_}@{PG_SERVER_}:{PG_PORT_}/'
 
-SQLALCHEMY_BASE_ADMIN_URI = f'postgresql+psycopg2://{PGA_ACCOUNT_}:{PGA_PASSWD_}@{PG_SERVER_}:{PG_PORT_}/'
+SQLALCHEMY_BASE_ADMIN_URI = f'postgresql+psycopg://{PGA_ACCOUNT_}:{PGA_PASSWD_}@{PG_SERVER_}:{PG_PORT_}/'
 
 
 class AuthenticationType(Enum):

source/app/datamgmt/case/case_db.py

@@ -161,7 +161,7 @@ def save_case_tags(tags, case):
 
 
 def get_case_tags(case_id):
-    case = Cases.query.get(case_id)
+    case = db.session.get(Cases, case_id)
 
     if case:
         return [tag.tag_title for tag in case.tags]

source/app/models/cases.py

@@ -95,10 +95,9 @@ def __init__(self,
                  state_id=None,
                  severity_id=None
                  ):
-        self.name = name[:200] if name else None,
-        self.soc_id = soc_id,
-        self.client_id = client_id,
-        self.description = description,
+        self.name = name[:200] if name else None
+        self.soc_id = soc_id
+        self.client_id = client_id
         self.user_id = iris_current_user.id if iris_current_user else user.id
         self.owner_id = self.user_id
         self.author = iris_current_user.user if iris_current_user else user.user
@@ -110,7 +109,7 @@ def __init__(self,
         self.case_uuid = uuid.uuid4()
         self.status_id = 0
         self.classification_id = classification_id
-        self.state_id = state_id,
+        self.state_id = state_id
         self.severity_id = severity_id
 
 

source/app/post_init.py

@@ -28,10 +28,7 @@
 
 from alembic import command
 from alembic.config import Config
-from sqlalchemy import create_engine
 from sqlalchemy import or_
-from sqlalchemy_utils import create_database
-from sqlalchemy_utils import database_exists
 
 from app import bc
 from app import celery
@@ -318,16 +315,34 @@ def connect_to_database(host: str, port: int) -> bool:
 
 
 def create_safe_db(url):
-    # Create a new engine object for the specified database
-    engine = create_engine(url)
+    """Create the target database if it does not already exist.
 
-    # Check if the database already exists
-    if not database_exists(engine.url):
-        # If the database does not exist, create it
-        create_database(engine.url)
+    sqlalchemy_utils.database_exists / create_database do not support the
+    psycopg3 driver (``postgresql+psycopg://``).  We replicate the minimal
+    behaviour using a raw psycopg3 connection so that no third-party helper
+    library is needed.
+    """
+    import psycopg
+    from sqlalchemy.engine import make_url
+
+    u = make_url(url)
+    db_name = u.database
 
-    # Dispose of the engine object
-    engine.dispose()
+    # Build a connection string to the postgres maintenance database so we can
+    # check / create the target DB without connecting to it first.
+    maintenance_url = u.set(database='postgres')
+    conninfo = (
+        f"host={maintenance_url.host} port={maintenance_url.port or 5432} "
+        f"dbname=postgres "
+        f"user={maintenance_url.username} password={maintenance_url.password}"
+    )
+
+    with psycopg.connect(conninfo, autocommit=True) as conn:
+        row = conn.execute(
+            "SELECT 1 FROM pg_database WHERE datname = %s", (db_name,)
+        ).fetchone()
+        if row is None:
+            conn.execute(f"CREATE DATABASE {db_name}")
 
 
 def create_safe_hooks():
@@ -981,12 +996,14 @@ def create_safe_tlp():
 
 def create_safe_server_settings(is_mfa_enabled):
     if not ServerSettings.query.count():
-        create_safe(db.session, ServerSettings,
-                    http_proxy="", https_proxy="", prevent_post_mod_repush=False,
-                    prevent_post_objects_repush=False,
-                    password_policy_min_length="12", password_policy_upper_case=True,
-                    password_policy_lower_case=True, password_policy_digit=True,
-                    password_policy_special_chars="", enforce_mfa=is_mfa_enabled)
+        srv = ServerSettings(
+            http_proxy="", https_proxy="", prevent_post_mod_repush=False,
+            prevent_post_objects_repush=False,
+            password_policy_min_length=12, password_policy_upper_case=True,
+            password_policy_lower_case=True, password_policy_digit=True,
+            password_policy_special_chars="", enforce_mfa=is_mfa_enabled)
+        db.session.add(srv)
+        db.session.commit()
 
 
 def create_safe_default_organisation():

source/app/schema/marshables.py

@@ -733,7 +733,7 @@ def verify_data(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]:
             assert_type_mml(input_var=data.get('asset_type_id'),
                             field_name="asset_type_id",
                             type=int)
-
+            data['asset_type_id'] = int(data['asset_type_id'])
             asset_type = AssetsType.query.filter(AssetsType.asset_id == data.get('asset_type_id')).count()
             if not asset_type:
                 raise ValidationError("Invalid asset type ID", field_name="asset_type_id")
@@ -742,6 +742,7 @@ def verify_data(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]:
             assert_type_mml(input_var=data.get('analysis_status_id'),
                             field_name="analysis_status_id", type=int,
                             allow_none=True)
+            data['analysis_status_id'] = int(data['analysis_status_id'])
 
         if data.get('analysis_status_id'):
             status = AnalysisStatus.query.filter(AnalysisStatus.id == data.get('analysis_status_id')).count()
@@ -973,6 +974,7 @@ def verify_data(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]:
         """
         if data.get('ioc_type_id'):
             assert_type_mml(input_var=data.get('ioc_type_id'), field_name="ioc_type_id", type=int)
+            data['ioc_type_id'] = int(data['ioc_type_id'])
             ioc_type = IocType.query.filter(IocType.type_id == data.get('ioc_type_id')).first()
             if not ioc_type:
                 raise ValidationError("Invalid IOC type ID", field_name="ioc_type_id")
@@ -986,7 +988,7 @@ def verify_data(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]:
         if data.get('ioc_tlp_id'):
             assert_type_mml(input_var=data.get('ioc_tlp_id'), field_name="ioc_tlp_id", type=int,
                             max_val=POSTGRES_INT_MAX)
-
+            data['ioc_tlp_id'] = int(data['ioc_tlp_id'])
             Tlp.query.filter(Tlp.tlp_id == data.get('ioc_tlp_id')).count()
 
         if data.get('ioc_tags'):
@@ -1065,6 +1067,7 @@ def verify_data(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]:
         """
         if data.get('ioc_type_id'):
             assert_type_mml(input_var=data.get('ioc_type_id'), field_name='ioc_type_id', type=int)
+            data['ioc_type_id'] = int(data['ioc_type_id'])
             ioc_type = IocType.query.filter(IocType.type_id == data.get('ioc_type_id')).first()
             if not ioc_type:
                 raise ValidationError('Invalid IOC type ID', field_name='ioc_type_id')
@@ -1078,7 +1081,7 @@ def verify_data(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]:
         if data.get('ioc_tlp_id'):
             assert_type_mml(input_var=data.get('ioc_tlp_id'), field_name='ioc_tlp_id', type=int,
                             max_val=POSTGRES_INT_MAX)
-
+            data['ioc_tlp_id'] = int(data['ioc_tlp_id'])
             Tlp.query.filter(Tlp.tlp_id == data.get('ioc_tlp_id')).count()
 
         if data.get('ioc_tags'):
@@ -1942,6 +1945,7 @@ def verify_data(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]:
         assert_type_mml(input_var=data.get('task_status_id'),
                         field_name='task_status_id',
                         type=int)
+        data['task_status_id'] = int(data['task_status_id'])
 
         status = TaskStatus.query.filter(TaskStatus.id == data.get('task_status_id')).count()
         if not status:

source/app/templates/layouts/default.html

@@ -74,9 +74,6 @@
 	<script src="/static/assets/js/plugin/tagsinput/suggesttag.js"></script>
 
 
-	<!-- jQuery UI -->
-	<script src="/static/assets/js/plugin/jquery-ui-1.12.1.custom/jquery-ui.min.js"></script>
-	<script src="/static/assets/js/plugin/jquery-ui-touch-punch/jquery.ui.touch-punch.min.js"></script>
 
 	<!-- jQuery Scrollbar -->
 	<script src="/static/assets/js/plugin/jquery-scrollbar/jquery.scrollbar.min.js"></script>
@@ -93,6 +90,9 @@
 	<!-- HTML 2 Canvas -->
 	<script src="/static/assets/js/plugin/html2canvas/html2canvas.min.js"></script>
 
+	<!-- Atlantis Draggable Shim -->
+	<script src="/static/assets/js/iris/atlantis-draggable-shim.js"></script>
+
 	<!-- Atlantis JS -->
 	<script src="/static/assets/js/atlantis.min.js"></script>
 

source/app/templates/layouts/default_centered.html

@@ -65,9 +65,6 @@
 	<script src="/static/assets/js/plugin/tagsinput/suggesttag.js"></script>
 
 
-	<!-- jQuery UI -->
-	<script src="/static/assets/js/plugin/jquery-ui-1.12.1.custom/jquery-ui.min.js"></script>
-	<script src="/static/assets/js/plugin/jquery-ui-touch-punch/jquery.ui.touch-punch.min.js"></script>
 
 	<!-- jQuery Scrollbar -->
 	<script src="/static/assets/js/plugin/jquery-scrollbar/jquery.scrollbar.min.js"></script>
@@ -83,6 +80,9 @@
 	<!-- HTML 2 Canvas -->
 	<script src="/static/assets/js/plugin/html2canvas/html2canvas.min.js"></script>
 
+	<!-- Atlantis Draggable Shim -->
+	<script src="/static/assets/js/iris/atlantis-draggable-shim.js"></script>
+
 	<!-- Atlantis JS -->
 	<script src="/static/assets/js/atlantis.min.js"></script>