diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 88cc293e9..c5cfa82d9 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -28,7 +28,7 @@ jobs: - name: Install aspects run: pip install -e . - name: Tutor config save - run: tutor config save + run: tutor config save --set ASPECTS_XAPI_SOURCE=ralph - name: Setup Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Free Disk Space (Ubuntu) @@ -101,7 +101,7 @@ jobs: - name: Install aspects run: pip install -e . - name: Tutor config save - run: tutor config save + run: tutor config save --set ASPECTS_XAPI_SOURCE=ralph - name: Setup Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Free Disk Space (Ubuntu) @@ -185,7 +185,7 @@ jobs: run: | pip install -r requirements/dev.txt pip install -e . - tutor config save + tutor config save --set ASPECTS_XAPI_SOURCE=ralph - name: Run Kubernetes tools uses: stefanprodan/kube-tools@49504f5a42faa937ed11d7705b995416f8e55271 # v1 with: @@ -305,6 +305,7 @@ jobs: print_pod_logs "$pod_name" done done + local-vector: runs-on: ubuntu-latest steps: diff --git a/README.rst b/README.rst index 80fe69713..8378ea14d 100644 --- a/README.rst +++ b/README.rst @@ -28,6 +28,32 @@ Compatibility Current versions of the plugin are compatible with Tutor 19.0.0 and later and support Open edX releases from Sumac onward. Older releases can support Open edX versions as far back as Nutmeg. Details are available in the `Aspects Documentation `_. +Breaking Changes +================ + +The default data pipeline has changed from Ralph to Vector improving performance and simplyfing the architecture. + +Key changes: + +- Vector is now the default for xAPI event ingestion +- The ``ASPECTS_VECTOR_RAW_XAPI_TABLE`` setting has been replaced with ``ASPECTS_RAW_XAPI_TABLE`` +- The default database has changed from ``xapi`` (Ralph) to ``openedx`` (Vector) + +To keep using Ralph as your data pipeline: + +.. code-block:: bash + + tutor config save --set ASPECTS_XAPI_SOURCE=ralph + tutor config save --set RUN_RALPH=True + tutor config save --set RUN_VECTOR=False + +This will configure Aspects to use Ralph with the ``xapi`` database, preserving your existing data. + +If you have customized ``ASPECTS_VECTOR_RAW_XAPI_TABLE`` in your configuration, update it to use ``ASPECTS_RAW_XAPI_TABLE`` instead. + +For new installations or users switching to Vector, your data will be stored in the ``openedx`` database. You can migrate existing data from the ``xapi`` database to ``openedx`` if needed. + + Installation ============ diff --git a/tutoraspects/commands_v1.py b/tutoraspects/commands_v1.py index 9eefcc907..6973e9f58 100644 --- a/tutoraspects/commands_v1.py +++ b/tutoraspects/commands_v1.py @@ -74,7 +74,6 @@ def dbt(only_changed: bool, command: string) -> list[tuple[str, str]]: return [ ( "aspects", - "echo 'Making dbt script executable...' && " f"echo 'Running dbt, only_changed: {only_changed} command: {command}' && " f"bash /app/aspects/scripts/dbt.sh {only_changed} {command} && " "echo 'Done!';", @@ -108,9 +107,7 @@ def alembic(command: string) -> list[tuple[str, str]]: return [ ( "aspects", - "echo 'Making dbt script executable...' && " - f"bash /app/aspects/scripts/alembic.sh {command} && " - "echo 'Done!';", + f"bash /app/aspects/scripts/alembic.sh {command} && " "echo 'Done!';", ), ] diff --git a/tutoraspects/patches/local-docker-compose-jobs-services b/tutoraspects/patches/local-docker-compose-jobs-services index e2d24601d..faac08606 100644 --- a/tutoraspects/patches/local-docker-compose-jobs-services +++ b/tutoraspects/patches/local-docker-compose-jobs-services @@ -16,6 +16,9 @@ aspects-job: volumes: - ../../env/plugins/aspects/apps/aspects:/app/aspects - ../../env/plugins/aspects/apps/aspects/scripts/:/app/aspects/scripts:ro + {%- for mount in iter_mounts(MOUNTS, "aspects-job") %} + - {{ mount }} + {%- endfor %} {% if RUN_CLICKHOUSE %} depends_on: - clickhouse diff --git a/tutoraspects/patches/xapi-db-load-config-yaml b/tutoraspects/patches/xapi-db-load-config-yaml index 7c347801d..fb1b5ab8f 100644 --- a/tutoraspects/patches/xapi-db-load-config-yaml +++ b/tutoraspects/patches/xapi-db-load-config-yaml @@ -1,6 +1,6 @@ # Ralph / ClickHouse backend configuration # ######################################## -backend: ralph_clickhouse +backend: vector db_host: {{ CLICKHOUSE_HOST }} db_port: {{ CLICKHOUSE_INTERNAL_HTTP_PORT }} db_name: {{ ASPECTS_XAPI_DATABASE }} @@ -11,9 +11,10 @@ lrs_username: "{{RALPH_LMS_USERNAME}}" lrs_password: "{{RALPH_LMS_PASSWORD}}" # Run options -log_dir: -num_batches: 3 +log_dir: logs +num_xapi_batches: 3 batch_size: 100 +num_workers: 4 # Overall start and end date for the entire run start_date: 2014-01-01 diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index 3d4cb13b3..0aa4d82d5 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -33,9 +33,9 @@ ("ASPECTS_VERSION", __version__), # For our default deployment we currently use Celery -> Ralph for transport, # so Vector is off by default. - ("RUN_VECTOR", False), + ("RUN_VECTOR", True), ("RUN_CLICKHOUSE", True), - ("RUN_RALPH", True), + ("RUN_RALPH", False), ("RUN_SUPERSET", True), ("DOCKER_IMAGE_ASPECTS", "edunext/aspects:{{ ASPECTS_VERSION }}"), ("DOCKER_IMAGE_CLICKHOUSE", "clickhouse/clickhouse-server:25.8"), @@ -61,7 +61,7 @@ # reasons. # Turn on event batching by default, performance is severely impacted by # turning this off. - ("EVENT_ROUTING_BACKEND_BATCHING_ENABLED", True), + ("EVENT_ROUTING_BACKEND_BATCHING_ENABLED", False), # Events are sent when they hit either the batch size or the batch interval # time limit (defaults here are 100 events or 5 seconds). # https://event-routing-backends.readthedocs.io/en/latest/getting_started.html#batching-configuration @@ -160,7 +160,7 @@ }, ), # ClickHouse xAPI settings - ("ASPECTS_XAPI_SOURCE", "ralph"), + ("ASPECTS_XAPI_SOURCE", "vector"), ( "ASPECTS_XAPI_DATABASE", """ @@ -181,7 +181,6 @@ ("ASPECTS_VECTOR_STORE_XAPI", True), ("ASPECTS_VECTOR_DATABASE", "openedx"), ("ASPECTS_VECTOR_RAW_TRACKING_LOGS_TABLE", "_tracking"), - ("ASPECTS_VECTOR_RAW_XAPI_TABLE", "xapi_events_all"), ("ASPECTS_DATA_TTL_EXPRESSION", "toDateTime(emission_time) + INTERVAL 1 YEAR"), ("ASPECTS_ALEMBIC_MIGRATIONS_DATABASE", "{{RALPH_DATABASE}}"), # Make sure LMS / CMS have event-routing-backends installed @@ -519,6 +518,11 @@ def _mount_superset_compose( """ if name == "superset": volumes += [("superset", "/app")] + elif name == "aspects-dbt": + volumes += [ + ("aspects-job", "/app/aspects-dbt"), + ("aspects-docs", "/app/aspects-dbt"), + ] return volumes diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0008_vector.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0008_vector.py index 0cf0cedce..cd8b9d65f 100644 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0008_vector.py +++ b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0008_vector.py @@ -24,7 +24,7 @@ def upgrade(): ) op.execute( f""" - CREATE TABLE IF NOT EXISTS {{ ASPECTS_VECTOR_DATABASE }}.{{ ASPECTS_VECTOR_RAW_XAPI_TABLE }} + CREATE TABLE IF NOT EXISTS {{ ASPECTS_VECTOR_DATABASE }}.{{ ASPECTS_RAW_XAPI_TABLE }} {on_cluster} ( event_id UUID, @@ -40,7 +40,7 @@ def upgrade(): def downgrade(): op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_VECTOR_DATABASE }}.{{ ASPECTS_VECTOR_RAW_XAPI_TABLE }}" + "DROP TABLE IF EXISTS {{ ASPECTS_VECTOR_DATABASE }}.{{ ASPECTS_RAW_XAPI_TABLE }}" f"{on_cluster}" ) op.execute( diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0011_vector_replacingmergetree_xapi_raw.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0011_vector_replacingmergetree_xapi_raw.py index d78d71a85..8ef68ddd9 100644 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0011_vector_replacingmergetree_xapi_raw.py +++ b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0011_vector_replacingmergetree_xapi_raw.py @@ -5,7 +5,7 @@ branch_labels = None depends_on = None -DESTINATION_TABLE = "{{ ASPECTS_VECTOR_DATABASE }}.{{ ASPECTS_VECTOR_RAW_XAPI_TABLE }}" +DESTINATION_TABLE = "{{ ASPECTS_VECTOR_DATABASE }}.{{ ASPECTS_RAW_XAPI_TABLE }}" TMP_TABLE_NEW = f"{DESTINATION_TABLE}_tmp_{revision}" TMP_TABLE_ORIG = f"{DESTINATION_TABLE}_tmp_mergetree_{revision}" on_cluster = ( diff --git a/tutoraspects/templates/aspects/apps/aspects/scripts/bootstrap.sh b/tutoraspects/templates/aspects/apps/aspects/scripts/bootstrap.sh index 9b426ca3b..12275e506 100644 --- a/tutoraspects/templates/aspects/apps/aspects/scripts/bootstrap.sh +++ b/tutoraspects/templates/aspects/apps/aspects/scripts/bootstrap.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash - set -eo pipefail if [ -z "${DBT_SSH_KEY+x}" ] @@ -14,31 +13,48 @@ then ssh-add /root/.ssh/id_rsa fi -export branch=$(git -C aspects-dbt/ describe --tags --exact-match 2>/dev/null || \ - git -C aspects-dbt/ branch --show-current 2>/dev/null || \ - git -C aspects-dbt/ rev-parse --short HEAD) -export repo=$(git -C aspects-dbt/ config --get remote.origin.url) -if [ "$DBT_BRANCH" != "$branch" ] || [ "$DBT_REPOSITORY" != "$repo" ]; -then - rm -rf aspects-dbt - echo "Current branch ${branch} != ${DBT_BRANCH}" - echo "Installing aspects-dbt" - echo "git clone -b ${DBT_BRANCH} ${DBT_REPOSITORY}" - git clone -b ${DBT_BRANCH} ${DBT_REPOSITORY} aspects-dbt - - cd aspects-dbt - - if [ -e "./requirements.txt" ] - then - echo "Installing dbt python requirements" - pip install -r ./requirements.txt +git config --global --add safe.directory '*' + +MOUNTED=false +if [ -d "aspects-dbt/.git" ]; then + if [ ! -f "aspects-dbt/.git/shallow" ]; then + MOUNTED=true + fi +fi + +current_branch=$(git -C aspects-dbt/ describe --tags --exact-match 2>/dev/null || \ + git -C aspects-dbt/ branch --show-current 2>/dev/null || \ + git -C aspects-dbt/ rev-parse --short HEAD) +current_repo=$(git -C aspects-dbt/ config --get remote.origin.url) + +if [ "$MOUNTED" = true ]; then + echo "Using mounted repo (branch: ${current_branch})" +else + # Only re-clone if branch/repo differ or directory is missing + if [ ! -d "aspects-dbt/.git" ] \ + || [ "${DBT_BRANCH}" != "${current_branch}" ] \ + || [ "${DBT_REPOSITORY}" != "${current_repo}" ]; then + + rm -rf aspects-dbt + + echo "Installing aspects-dbt" + echo "git clone -b ${DBT_BRANCH} ${DBT_REPOSITORY}" + git clone -b "${DBT_BRANCH}" "${DBT_REPOSITORY}" aspects-dbt else - echo "No requirements.txt file found; skipping" + echo "Using existing cloned repo (branch: ${current_branch})" fi +fi - echo "Installing dbt dependencies" - dbt deps +cd aspects-dbt +if [ -e "./requirements.txt" ]; then + echo "Installing dbt python requirements" + uv pip install -r ./requirements.txt --system +else + echo "No requirements.txt file found; skipping" fi -mkdir -p $DBT_STATE +echo "Installing dbt dependencies" +dbt deps + +mkdir -p "${DBT_STATE}" diff --git a/tutoraspects/templates/aspects/apps/vector/k8s.toml b/tutoraspects/templates/aspects/apps/vector/k8s.toml index 025aa357a..8af523232 100644 --- a/tutoraspects/templates/aspects/apps/vector/k8s.toml +++ b/tutoraspects/templates/aspects/apps/vector/k8s.toml @@ -8,6 +8,6 @@ extra_namespace_label_selector = "kubernetes.io/metadata.name={{ K8S_NAMESPACE } [transforms.openedx_containers] type = "filter" inputs = ["kubernetes_logs"] -condition = '.kubernetes.pod_namespace == "{{ K8S_NAMESPACE }}" && includes(["lms", "cms", "lms-job", "cms-job", "lms-worker", "cms-worker"], .kubernetes.container_name)' +condition = '.kubernetes.pod_namespace == "{{ K8S_NAMESPACE }}" && includes(["lms", "cms", "lms-worker", "cms-worker", "lms-job", "cms-job", "aspects-job", "aspects-consumer"], .kubernetes.container_name)' {% include "aspects/apps/vector/partials/common-post.toml" %} diff --git a/tutoraspects/templates/aspects/apps/vector/local.toml b/tutoraspects/templates/aspects/apps/vector/local.toml index b0131b1f3..ccbbdc816 100644 --- a/tutoraspects/templates/aspects/apps/vector/local.toml +++ b/tutoraspects/templates/aspects/apps/vector/local.toml @@ -8,6 +8,6 @@ type = "docker_logs" [transforms.openedx_containers] type = "filter" inputs = ["docker_logs"] -condition = 'includes(["lms", "cms", "lms-worker", "cms-worker", "lms-job", "cms-job"], .label."com.docker.compose.service")' +condition = 'includes(["lms", "cms", "lms-worker", "cms-worker", "lms-job", "cms-job", "aspects-job", "aspects-consumer"], .label."com.docker.compose.service")' {% include "aspects/apps/vector/partials/common-post.toml" %} diff --git a/tutoraspects/templates/aspects/apps/vector/partials/common-post.toml b/tutoraspects/templates/aspects/apps/vector/partials/common-post.toml index 6bce1cb0b..dc2d91216 100644 --- a/tutoraspects/templates/aspects/apps/vector/partials/common-post.toml +++ b/tutoraspects/templates/aspects/apps/vector/partials/common-post.toml @@ -125,7 +125,7 @@ date_time_best_effort = true inputs = ["xapi"] endpoint = "{% if CLICKHOUSE_SECURE_CONNECTION %}https{% else %}http{% endif %}://{{ CLICKHOUSE_HOST }}:{{ CLICKHOUSE_INTERNAL_HTTP_PORT }}" database = "{{ ASPECTS_VECTOR_DATABASE }}" -table = "{{ ASPECTS_VECTOR_RAW_XAPI_TABLE }}" +table = "{{ ASPECTS_RAW_XAPI_TABLE }}" healthcheck = false {% endif %} diff --git a/tutoraspects/templates/aspects/build/aspects/Dockerfile b/tutoraspects/templates/aspects/build/aspects/Dockerfile index eb8885f30..ccfe6b24b 100644 --- a/tutoraspects/templates/aspects/build/aspects/Dockerfile +++ b/tutoraspects/templates/aspects/build/aspects/Dockerfile @@ -2,16 +2,15 @@ FROM python:3.12 WORKDIR /app +COPY --from=ghcr.io/astral-sh/uv:0.10.8 /uv /uvx /bin/ +ENV PATH="/root/.local/bin/:$PATH" COPY ./requirements.txt /app/requirements.txt -RUN pip install -r /app/requirements.txt - -RUN git clone -b {{ DBT_BRANCH }} {{ DBT_REPOSITORY }} aspects-dbt +RUN uv pip install -r /app/requirements.txt --system +RUN git clone --depth 1 -b {{ DBT_BRANCH }} {{ DBT_REPOSITORY }} aspects-dbt WORKDIR /app/aspects-dbt - -RUN pip install -r requirements.txt - +RUN uv pip install -r requirements.txt --system RUN dbt deps WORKDIR /app diff --git a/tutoraspects/templates/aspects/build/aspects/requirements.txt b/tutoraspects/templates/aspects/build/aspects/requirements.txt index bbbf5e7c1..b018e615d 100644 --- a/tutoraspects/templates/aspects/build/aspects/requirements.txt +++ b/tutoraspects/templates/aspects/build/aspects/requirements.txt @@ -1,5 +1,5 @@ # alembic packages alembic==1.14.1 clickhouse-sqlalchemy==0.3.2 -git+https://github.com/openedx/xapi-db-load@1.5.0 +git+https://github.com/openedx/xapi-db-load@3.1.0 pyyaml