From 864560d6ad11a5bccc90110b9cefcf7214e71bf0 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Wed, 25 Feb 2026 18:45:30 +0800 Subject: [PATCH 01/68] feat(ai): implement versioning and finalization for AI comments --- ...60225120000_create_ai_comment_versions.exs | 21 +++++++++++++++++++ ...260225120001_add_fields_to_ai_comments.exs | 14 +++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 priv/repo/migrations/20260225120000_create_ai_comment_versions.exs create mode 100644 priv/repo/migrations/20260225120001_add_fields_to_ai_comments.exs diff --git a/priv/repo/migrations/20260225120000_create_ai_comment_versions.exs b/priv/repo/migrations/20260225120000_create_ai_comment_versions.exs new file mode 100644 index 000000000..a31e2d7d9 --- /dev/null +++ b/priv/repo/migrations/20260225120000_create_ai_comment_versions.exs @@ -0,0 +1,21 @@ +defmodule Cadet.Repo.Migrations.CreateAiCommentVersions do + use Ecto.Migration + + def change do + create table(:ai_comment_versions) do + add :ai_comment_id, references(:ai_comment_logs, on_delete: :delete_all), null: false + add :comment_index, :integer, null: false + add :version_number, :integer, null: false, default: 1 + add :editor_id, references(:users, on_delete: :nilify_all) + add :content, :text + add :diff_json, :map + add :diff_unified, :text + + timestamps() + end + + create index(:ai_comment_versions, [:ai_comment_id]) + create index(:ai_comment_versions, [:editor_id]) + create unique_index(:ai_comment_versions, [:ai_comment_id, :comment_index, :version_number]) + end +end diff --git a/priv/repo/migrations/20260225120001_add_fields_to_ai_comments.exs b/priv/repo/migrations/20260225120001_add_fields_to_ai_comments.exs new file mode 100644 index 000000000..82a50a4e8 --- /dev/null +++ b/priv/repo/migrations/20260225120001_add_fields_to_ai_comments.exs @@ -0,0 +1,14 @@ +defmodule Cadet.Repo.Migrations.AddFieldsToAiComments do + use Ecto.Migration + + def change do + alter table(:ai_comment_logs) do + add :selected_indices, {:array, :integer} + add :finalized_by_id, references(:users, on_delete: :nilify_all) + add :finalized_at, :utc_datetime + end + + create index(:ai_comment_logs, [:answer_id]) + create index(:ai_comment_logs, [:finalized_by_id]) + end +end From 88593a1804d96f9f8b740cc35856a83e424cae24 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Wed, 25 Feb 2026 21:32:57 +0800 Subject: [PATCH 02/68] feat(ai_comments): add comment selection and word-level version tracking --- lib/cadet/ai_comments.ex | 75 +++++++++- lib/cadet/ai_comments/ai_comment.ex | 7 +- lib/cadet/ai_comments/ai_comment_version.ex | 34 +++++ .../controllers/generate_ai_comments.ex | 135 ++++++++++++++++++ ...60225120000_create_ai_comment_versions.exs | 20 +-- ...260225120001_add_fields_to_ai_comments.exs | 10 +- 6 files changed, 264 insertions(+), 17 deletions(-) create mode 100644 lib/cadet/ai_comments/ai_comment_version.ex diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index e37133b98..24cd68c79 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -5,7 +5,7 @@ defmodule Cadet.AIComments do import Ecto.Query alias Cadet.Repo - alias Cadet.AIComments.AIComment + alias Cadet.AIComments.{AIComment, AICommentVersion} @doc """ Creates a new AI comment log entry. @@ -74,4 +74,77 @@ defmodule Cadet.AIComments do |> Repo.update() end end + + @doc """ + Saves selected comment indices and finalization metadata for an AI comment. + """ + def save_selected_comments(answer_id, selected_indices, finalized_by_id) do + case get_latest_ai_comment(answer_id) do + nil -> + {:error, :not_found} + + comment -> + comment + |> AIComment.changeset(%{ + selected_indices: selected_indices, + finalized_by_id: finalized_by_id, + finalized_at: DateTime.truncate(DateTime.utc_now(), :second) + }) + |> Repo.update() + end + end + + @doc """ + Creates a new version entry for a specific comment index. + Automatically determines the next version number. + """ + def create_comment_version(ai_comment_id, comment_index, content, editor_id, diff \\ %{}) do + next_version = + Repo.one( + from(v in AICommentVersion, + where: v.ai_comment_id == ^ai_comment_id and v.comment_index == ^comment_index, + select: coalesce(max(v.version_number), 0) + ) + ) + 1 + + %AICommentVersion{} + |> AICommentVersion.changeset( + Map.merge( + %{ + ai_comment_id: ai_comment_id, + comment_index: comment_index, + version_number: next_version, + content: content, + editor_id: editor_id + }, + diff + ) + ) + |> Repo.insert() + end + + @doc """ + Gets all versions for a specific AI comment, ordered by comment_index and version_number. + """ + def get_comment_versions(ai_comment_id) do + Repo.all( + from(v in AICommentVersion, + where: v.ai_comment_id == ^ai_comment_id, + order_by: [asc: v.comment_index, asc: v.version_number] + ) + ) + end + + @doc """ + Gets the latest version for a specific comment index. + """ + def get_latest_version(ai_comment_id, comment_index) do + Repo.one( + from(v in AICommentVersion, + where: v.ai_comment_id == ^ai_comment_id and v.comment_index == ^comment_index, + order_by: [desc: v.version_number], + limit: 1 + ) + ) + end end diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex index 64d5d4cfe..e986f739b 100644 --- a/lib/cadet/ai_comments/ai_comment.ex +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -12,14 +12,19 @@ defmodule Cadet.AIComments.AIComment do field(:response, :string) field(:error, :string) field(:final_comment, :string) + field(:selected_indices, {:array, :integer}) + field(:finalized_at, :utc_datetime) belongs_to(:answer, Cadet.Assessments.Answer) + belongs_to(:finalized_by, Cadet.Accounts.User, foreign_key: :finalized_by_id) + + has_many(:versions, Cadet.AIComments.AICommentVersion) timestamps() end @required_fields ~w(answer_id raw_prompt answers_json)a - @optional_fields ~w(response error final_comment)a + @optional_fields ~w(response error final_comment selected_indices finalized_by_id finalized_at)a def changeset(ai_comment, attrs) do ai_comment diff --git a/lib/cadet/ai_comments/ai_comment_version.ex b/lib/cadet/ai_comments/ai_comment_version.ex new file mode 100644 index 000000000..c120c68c3 --- /dev/null +++ b/lib/cadet/ai_comments/ai_comment_version.ex @@ -0,0 +1,34 @@ +defmodule Cadet.AIComments.AICommentVersion do + @moduledoc """ + Defines the schema and changeset for AI comment versions. + Tracks per-comment edits made by tutors. + """ + + use Ecto.Schema + import Ecto.Changeset + + schema "ai_comment_versions" do + field(:comment_index, :integer) + field(:version_number, :integer) + field(:content, :string) + field(:diff_json, :map) + field(:diff_unified, :string) + + belongs_to(:ai_comment, Cadet.AIComments.AIComment) + belongs_to(:editor, Cadet.Accounts.User, foreign_key: :editor_id) + + timestamps() + end + + @required_fields ~w(ai_comment_id comment_index version_number content)a + @optional_fields ~w(editor_id diff_json diff_unified)a + + def changeset(version, attrs) do + version + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) + |> foreign_key_constraint(:ai_comment_id) + |> foreign_key_constraint(:editor_id) + |> unique_constraint([:ai_comment_id, :comment_index, :version_number]) + end +end diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 24122dd4d..68232f0eb 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -291,6 +291,100 @@ defmodule CadetWeb.AICodeAnalysisController do end end + @doc """ + Saves the chosen comment indices and optional edits for each selected comment. + Expects: selected_indices (list of ints), edits (optional map of index => edited_text). + """ + def save_chosen_comments( + conn, + params = %{ + "submissionid" => _submission_id, + "questionid" => _question_id, + "answer_id" => answer_id, + "selected_indices" => selected_indices + } + ) do + editor_id = conn.assigns.course_reg.user_id + edits = Map.get(params, "edits", %{}) + + with ai_comment when not is_nil(ai_comment) <- AIComments.get_latest_ai_comment(answer_id), + {:ok, _updated} <- + AIComments.save_selected_comments(answer_id, selected_indices, editor_id) do + # Split the original response into individual comments + original_comments = + (ai_comment.response || "") + |> String.split("|||") + |> Enum.map(&String.trim/1) + |> Enum.filter(&(&1 != "")) + + # Create version entries for each edit + version_results = + Enum.map(edits, fn {index_str, edited_text} -> + index = String.to_integer(index_str) + original = Enum.at(original_comments, index, "") + diff = compute_diff(original, edited_text) + + AIComments.create_comment_version( + ai_comment.id, + index, + edited_text, + editor_id, + diff + ) + end) + + errors = Enum.filter(version_results, &match?({:error, _}, &1)) + + if errors == [] do + json(conn, %{"status" => "success"}) + else + conn + |> put_status(:unprocessable_entity) + |> text("Failed to save some comment versions") + end + else + nil -> + conn + |> put_status(:not_found) + |> text("AI comment not found for this answer") + + {:error, _} -> + conn + |> put_status(:unprocessable_entity) + |> text("Failed to save chosen comments") + end + end + + defp compute_diff(original, edited) do + original_tokens = tokenize(original) + edited_tokens = tokenize(edited) + + diff = List.myers_difference(original_tokens, edited_tokens) + + ops = + diff + |> Enum.flat_map(fn + {:eq, tokens} -> [%{op: "eq", text: Enum.join(tokens)}] + {:del, tokens} -> [%{op: "del", text: Enum.join(tokens)}] + {:ins, tokens} -> [%{op: "add", text: Enum.join(tokens)}] + end) + + unified = + Enum.map_join(ops, fn + %{op: "eq", text: text} -> " " <> text + %{op: "del", text: text} -> "-" <> text + %{op: "add", text: text} -> "+" <> text + end) + + %{diff_json: %{"ops" => ops}, diff_unified: unified} + end + + # Splits text into tokens at word boundaries, preserving whitespace and punctuation + # as separate tokens so the diff is word-level accurate. + defp tokenize(text) do + Regex.split(~r/\b/, text, include_captures: false, trim: true) + end + swagger_path :generate_ai_comments do post("/courses/{course_id}/admin/generate-comments/{answer_id}") @@ -335,6 +429,29 @@ defmodule CadetWeb.AICodeAnalysisController do response(403, "Forbidden") end + swagger_path :save_chosen_comments do + post("/courses/{course_id}/admin/save-chosen-comments/{submissionid}/{questionid}") + + summary("Save chosen comment indices and optional edits for a submission.") + + security([%{JWT: []}]) + + consumes("application/json") + produces("application/json") + + parameters do + course_id(:path, :integer, "course id", required: true) + submissionid(:path, :integer, "submission id", required: true) + questionid(:path, :integer, "question id", required: true) + + body(:body, Schema.ref(:SaveChosenCommentsBody), "Chosen comments payload", required: true) + end + + response(200, "OK", Schema.ref(:SaveChosenComments)) + response(404, "AI comment not found") + response(422, "Failed to save") + end + def swagger_definitions do %{ GenerateAIComments: @@ -344,6 +461,24 @@ defmodule CadetWeb.AICodeAnalysisController do end end, SaveFinalComment: + swagger_schema do + properties do + status(:string, "Status of the operation") + end + end, + SaveChosenCommentsBody: + swagger_schema do + properties do + answer_id(:integer, "The answer ID", required: true) + + selected_indices(Schema.ref(:IntegerArray), "Indices of chosen comments", + required: true + ) + + edits(:object, "Map of comment index to edited text") + end + end, + SaveChosenComments: swagger_schema do properties do status(:string, "Status of the operation") diff --git a/priv/repo/migrations/20260225120000_create_ai_comment_versions.exs b/priv/repo/migrations/20260225120000_create_ai_comment_versions.exs index a31e2d7d9..04f70e480 100644 --- a/priv/repo/migrations/20260225120000_create_ai_comment_versions.exs +++ b/priv/repo/migrations/20260225120000_create_ai_comment_versions.exs @@ -3,19 +3,19 @@ defmodule Cadet.Repo.Migrations.CreateAiCommentVersions do def change do create table(:ai_comment_versions) do - add :ai_comment_id, references(:ai_comment_logs, on_delete: :delete_all), null: false - add :comment_index, :integer, null: false - add :version_number, :integer, null: false, default: 1 - add :editor_id, references(:users, on_delete: :nilify_all) - add :content, :text - add :diff_json, :map - add :diff_unified, :text + add(:ai_comment_id, references(:ai_comment_logs, on_delete: :delete_all), null: false) + add(:comment_index, :integer, null: false) + add(:version_number, :integer, null: false, default: 1) + add(:editor_id, references(:users, on_delete: :nilify_all)) + add(:content, :text) + add(:diff_json, :map) + add(:diff_unified, :text) timestamps() end - create index(:ai_comment_versions, [:ai_comment_id]) - create index(:ai_comment_versions, [:editor_id]) - create unique_index(:ai_comment_versions, [:ai_comment_id, :comment_index, :version_number]) + create(index(:ai_comment_versions, [:ai_comment_id])) + create(index(:ai_comment_versions, [:editor_id])) + create(unique_index(:ai_comment_versions, [:ai_comment_id, :comment_index, :version_number])) end end diff --git a/priv/repo/migrations/20260225120001_add_fields_to_ai_comments.exs b/priv/repo/migrations/20260225120001_add_fields_to_ai_comments.exs index 82a50a4e8..4b7e8e672 100644 --- a/priv/repo/migrations/20260225120001_add_fields_to_ai_comments.exs +++ b/priv/repo/migrations/20260225120001_add_fields_to_ai_comments.exs @@ -3,12 +3,12 @@ defmodule Cadet.Repo.Migrations.AddFieldsToAiComments do def change do alter table(:ai_comment_logs) do - add :selected_indices, {:array, :integer} - add :finalized_by_id, references(:users, on_delete: :nilify_all) - add :finalized_at, :utc_datetime + add(:selected_indices, {:array, :integer}) + add(:finalized_by_id, references(:users, on_delete: :nilify_all)) + add(:finalized_at, :utc_datetime) end - create index(:ai_comment_logs, [:answer_id]) - create index(:ai_comment_logs, [:finalized_by_id]) + create(index(:ai_comment_logs, [:answer_id])) + create(index(:ai_comment_logs, [:finalized_by_id])) end end From 3190d3e9649ed9880fd34233c3f3e2360d2b6c93 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Thu, 26 Feb 2026 00:19:06 +0800 Subject: [PATCH 03/68] dependencies --- .tool-versions | 2 + config/cadet.exs | 179 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 .tool-versions create mode 100644 config/cadet.exs diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 000000000..43e97011d --- /dev/null +++ b/.tool-versions @@ -0,0 +1,2 @@ +elixir 1.18.3-otp-27 +erlang 27.3.4 diff --git a/config/cadet.exs b/config/cadet.exs new file mode 100644 index 000000000..f20aae580 --- /dev/null +++ b/config/cadet.exs @@ -0,0 +1,179 @@ +# Example production configuration file + +import Config + +# See https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-runtime-configuration +# except for cors_endpoints, load_from_system_env which are custom +config :cadet, CadetWeb.Endpoint, + # See https://hexdocs.pm/corsica/Corsica.html#module-origins + # Remove for "*" + cors_endpoints: "example.com", + server: true, + # If true, expects an environment variable PORT specifying the port to listen on + load_from_system_env: true, + url: [host: "api.example.com", port: 80], + # You can specify the port here instead + # e.g http: [compress: true, port: 4000] + http: [compress: true], + # Generate using `mix phx.gen.secret` + secret_key_base: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + +config :cadet, Cadet.Auth.Guardian, + issuer: "cadet", + # Generate using `mix phx.gen.secret` + secret_key: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + +config :cadet, Cadet.Repo, + # Do not change this, only Postgres is supported + # (This is here because of how configuration works in Elixir.) + adapter: Ecto.Adapters.Postgres, + # The AWS Secrets Manager secret name containing the database connection details + rds_secret_name: "-cadet-db", + # Alternatively, you can include the credentials here: + # (comment out or remove rds_secret_name) + # username: "postgres", + # password: "postgres", + # database: "cadet_stg", + # hostname: "localhost", + pool_size: 10 + +config :cadet, + identity_providers: %{ + # # To use authentication with ADFS. + # "nusnet" => + # {Cadet.Auth.Providers.ADFS, + # %{ + # # The OAuth2 token endpoint. + # token_endpoint: "https://my-adfs/adfs/oauth2/token" + # }}, + # # An example of OpenID authentication with Cognito. Any OpenID-compliant + # # provider should work. + # "cognito" => + # {Cadet.Auth.Providers.OpenID, + # %{ + # # This should match a key in openid_connect_providers below + # openid_provider: :cognito, + # # You may need to write your own claim extractor for other providers + # claim_extractor: Cadet.Auth.Providers.CognitoClaimExtractor + # }}, + + # # Example SAML authentication with NUS Student IdP + # "test_saml" => + # {Cadet.Auth.Providers.SAML, + # %{ + # assertion_extractor: Cadet.Auth.Providers.NusstuAssertionExtractor, + # client_redirect_url: "http://cadet.frontend:8000/login/callback" + # }}, + + "test" => + {Cadet.Auth.Providers.Config, + [ + %{ + token: "admin_token", + code: "admin_code", + name: "Test Admin", + username: "admin", + role: :admin + }, + %{ + token: "staff_token", + code: "staff_code", + name: "Test Staff", + username: "staff", + role: :staff + }, + %{ + token: "student_token", + code: "student_code", + name: "Test Student", + username: "student", + role: :student + } + ]} + }, + # See https://hexdocs.pm/openid_connect/readme.html + # openid_connect_providers: [ + # cognito: [ + # discovery_document_uri: "", + # client_id: "", + # client_secret: "", + # response_type: "code", + # scope: "openid profile" + # ] + # ], + autograder: [ + lambda_name: "-grader" + ], + uploader: [ + assets_bucket: "-assets", + assets_prefix: "courses/", + sourcecasts_bucket: "-cadet-sourcecasts" + ], + # Configuration for Sling integration (executing on remote devices) + remote_execution: [ + # Prefix for AWS IoT thing names + thing_prefix: "-sling", + # AWS IoT thing group to put created things into (must be set-up beforehand) + thing_group: "-sling", + # Role ARN to use when generating signed client MQTT-WS URLs (must be set-up beforehand) + # Note, you need to specify the correct account ID. Find it in AWS IAM at the bottom left. + client_role_arn: "arn:aws:iam:::role/-cadet-frontend" + ] + +# Sentry DSN. This is only really useful to the NUS SoC deployments, but you can +# specify a DSN here if you wish to track backend errors. +# config :sentry, +# dsn: "https://public_key/sentry.io/somethingsomething" + +# If you are not running on EC2, you will need to configure an AWS access token +# for the backend to access AWS resources: +# +# This will make ExAWS read the values from the corresponding environment variables. +# config :ex_aws, +# access_key_id: [{:system, "AWS_ACCESS_KEY_ID"}], +# secret_access_key: [{:system, "AWS_SECRET_ACCESS_KEY"}] +# +# You can also just specify the values directly: +# config :ex_aws, +# access_key_id: "ACCESS KEY", +# secret_access_key: "SECRET KEY" +# +# You may also want to change the AWS or S3 region used for all resources: +# (Note, the default is ap-southeast-1 i.e. Singapore) +# config :ex_aws, +# region: "ap-southeast-1", +# s3: [ +# scheme: "https://", +# host: "s3.ap-southeast-1.amazonaws.com", +# region: "ap-southeast-1" +# ] + +# You may also want to change the timezone used for scheduled jobs +# config :cadet, Cadet.Jobs.Scheduler, +# timezone: "Asia/Singapore", + +# # Additional configuration for SAML authentication +# # For more details, see https://github.com/handnot2/samly +# config :samly, Samly.Provider, +# idp_id_from: :path_segment, +# service_providers: [ +# %{ +# id: "source-academy-backend", +# certfile: "example_path/certfile.pem", +# keyfile: "example_path/keyfile.pem" +# } +# ], +# identity_providers: [ +# %{ +# id: "student", +# sp_id: "source-academy-backend", +# base_url: "https://example_backend/sso", +# metadata_file: "student_idp_metadata.xml" +# }, +# %{ +# id: "staff", +# sp_id: "source-academy-backend", +# base_url: "https://example_backend/sso", +# metadata_file: "staff_idp_metadata.xml" +# } +# ] From 025938d5deae2026f38f950b0a54bb2f54174d2b Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Thu, 26 Feb 2026 00:19:45 +0800 Subject: [PATCH 04/68] dependencies --- .tool-versions | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .tool-versions diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 000000000..0ecc2753e --- /dev/null +++ b/.tool-versions @@ -0,0 +1,4 @@ +elixir 1.19.0-otp-27 +erlang 27.2.4 +local 1.19.0-otp-27 +exlixir 1.19.0-otp-27 From 30ad398133894306004a82d0c9874f79cd3afbde Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 3 Mar 2026 17:58:33 +0800 Subject: [PATCH 05/68] feat: add LLM usage logs and feedback schema --- .../20260303120000_create_llm_usage_logs.exs | 21 +++++++++++++++++++ .../20260303120001_create_llm_feedback.exs | 19 +++++++++++++++++ ...120002_add_question_id_to_llm_feedback.exs | 11 ++++++++++ 3 files changed, 51 insertions(+) create mode 100644 priv/repo/migrations/20260303120000_create_llm_usage_logs.exs create mode 100644 priv/repo/migrations/20260303120001_create_llm_feedback.exs create mode 100644 priv/repo/migrations/20260303120002_add_question_id_to_llm_feedback.exs diff --git a/priv/repo/migrations/20260303120000_create_llm_usage_logs.exs b/priv/repo/migrations/20260303120000_create_llm_usage_logs.exs new file mode 100644 index 000000000..ce8c16591 --- /dev/null +++ b/priv/repo/migrations/20260303120000_create_llm_usage_logs.exs @@ -0,0 +1,21 @@ +defmodule Cadet.Repo.Migrations.CreateLlmUsageLogs do + use Ecto.Migration + + def change do + create table(:llm_usage_logs) do + add(:course_id, references(:courses, on_delete: :delete_all), null: false) + add(:assessment_id, references(:assessments, on_delete: :delete_all), null: false) + add(:question_id, references(:questions, on_delete: :delete_all), null: false) + add(:answer_id, references(:answers, on_delete: :delete_all), null: false) + add(:submission_id, references(:submissions, on_delete: :delete_all), null: false) + add(:user_id, references(:users, on_delete: :nilify_all), null: false) + + timestamps() + end + + create(index(:llm_usage_logs, [:course_id])) + create(index(:llm_usage_logs, [:assessment_id])) + create(index(:llm_usage_logs, [:user_id])) + create(index(:llm_usage_logs, [:submission_id])) + end +end diff --git a/priv/repo/migrations/20260303120001_create_llm_feedback.exs b/priv/repo/migrations/20260303120001_create_llm_feedback.exs new file mode 100644 index 000000000..b086c61a0 --- /dev/null +++ b/priv/repo/migrations/20260303120001_create_llm_feedback.exs @@ -0,0 +1,19 @@ +defmodule Cadet.Repo.Migrations.CreateLlmFeedback do + use Ecto.Migration + + def change do + create table(:llm_feedback) do + add(:course_id, references(:courses, on_delete: :delete_all), null: false) + add(:assessment_id, references(:assessments, on_delete: :delete_all)) + add(:user_id, references(:users, on_delete: :nilify_all), null: false) + add(:rating, :integer) + add(:body, :text, null: false) + + timestamps() + end + + create(index(:llm_feedback, [:course_id])) + create(index(:llm_feedback, [:assessment_id])) + create(index(:llm_feedback, [:user_id])) + end +end diff --git a/priv/repo/migrations/20260303120002_add_question_id_to_llm_feedback.exs b/priv/repo/migrations/20260303120002_add_question_id_to_llm_feedback.exs new file mode 100644 index 000000000..72e7bdbc7 --- /dev/null +++ b/priv/repo/migrations/20260303120002_add_question_id_to_llm_feedback.exs @@ -0,0 +1,11 @@ +defmodule Cadet.Repo.Migrations.AddQuestionIdToLlmFeedback do + use Ecto.Migration + + def change do + alter table(:llm_feedback) do + add(:question_id, references(:questions, on_delete: :delete_all)) + end + + create(index(:llm_feedback, [:question_id])) + end +end From dcd39c2c8e35103f7257027855fa213465c69da3 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 3 Mar 2026 17:59:28 +0800 Subject: [PATCH 06/68] feat: implement LLM stats context and models --- lib/cadet/llm_stats.ex | 164 +++++++++++++++++++++++++++ lib/cadet/llm_stats/llm_feedback.ex | 34 ++++++ lib/cadet/llm_stats/llm_usage_log.ex | 33 ++++++ 3 files changed, 231 insertions(+) create mode 100644 lib/cadet/llm_stats.ex create mode 100644 lib/cadet/llm_stats/llm_feedback.ex create mode 100644 lib/cadet/llm_stats/llm_usage_log.ex diff --git a/lib/cadet/llm_stats.ex b/lib/cadet/llm_stats.ex new file mode 100644 index 000000000..0987af480 --- /dev/null +++ b/lib/cadet/llm_stats.ex @@ -0,0 +1,164 @@ +defmodule Cadet.LLMStats do + @moduledoc """ + Context module for LLM usage statistics and feedback. + Provides per-assessment and per-question statistics and feedback management. + """ + + import Ecto.Query + alias Cadet.Repo + alias Cadet.LLMStats.{LLMUsageLog, LLMFeedback} + + # ===================== + # Usage Logging + # ===================== + + @doc """ + Logs a usage event when "Generate Comments" is invoked. + """ + def log_usage(attrs) do + %LLMUsageLog{} + |> LLMUsageLog.changeset(attrs) + |> Repo.insert() + end + + # ===================== + # Assessment-level Statistics + # ===================== + + @doc """ + Returns LLM usage statistics for a specific assessment. + + Returns: + - total_uses: total "Generate Comments" invocations + - unique_submissions: unique submissions that had LLM used + - unique_users: unique users who used the feature + - questions: per-question breakdown with stats + """ + def get_assessment_statistics(course_id, assessment_id) do + base = + from(l in LLMUsageLog, + where: l.course_id == ^course_id and l.assessment_id == ^assessment_id + ) + + total_uses = Repo.aggregate(base, :count) + + unique_submissions = + Repo.one( + from(l in base, + select: count(l.submission_id, :distinct) + ) + ) + + unique_users = + Repo.one( + from(l in base, + select: count(l.user_id, :distinct) + ) + ) + + # Per-question breakdown + questions = + Repo.all( + from(l in LLMUsageLog, + join: q in assoc(l, :question), + where: l.course_id == ^course_id and l.assessment_id == ^assessment_id, + group_by: [q.id, q.display_order], + select: %{ + question_id: q.id, + display_order: q.display_order, + total_uses: count(l.id), + unique_submissions: count(l.submission_id, :distinct), + unique_users: count(l.user_id, :distinct) + }, + order_by: [asc: q.display_order] + ) + ) + + %{ + total_uses: total_uses, + unique_submissions: unique_submissions, + unique_users: unique_users, + questions: questions + } + end + + # ===================== + # Question-level Statistics + # ===================== + + @doc """ + Returns LLM usage statistics for a specific question within an assessment. + """ + def get_question_statistics(course_id, assessment_id, question_id) do + base = + from(l in LLMUsageLog, + where: + l.course_id == ^course_id and l.assessment_id == ^assessment_id and + l.question_id == ^question_id + ) + + total_uses = Repo.aggregate(base, :count) + + unique_submissions = + Repo.one( + from(l in base, + select: count(l.submission_id, :distinct) + ) + ) + + unique_users = + Repo.one( + from(l in base, + select: count(l.user_id, :distinct) + ) + ) + + %{ + total_uses: total_uses, + unique_submissions: unique_submissions, + unique_users: unique_users + } + end + + # ===================== + # Feedback + # ===================== + + @doc """ + Submits user feedback for the LLM feature. + """ + def submit_feedback(attrs) do + %LLMFeedback{} + |> LLMFeedback.changeset(attrs) + |> Repo.insert() + end + + @doc """ + Gets feedback for an assessment, optionally filtered by question_id. + """ + def get_feedback(course_id, assessment_id, question_id \\ nil) do + query = + from(f in LLMFeedback, + join: u in assoc(f, :user), + where: f.course_id == ^course_id and f.assessment_id == ^assessment_id, + order_by: [desc: f.inserted_at], + select: %{ + id: f.id, + rating: f.rating, + body: f.body, + user_name: u.name, + question_id: f.question_id, + inserted_at: f.inserted_at + } + ) + + query = + if question_id do + from(f in query, where: f.question_id == ^question_id) + else + query + end + + Repo.all(query) + end +end diff --git a/lib/cadet/llm_stats/llm_feedback.ex b/lib/cadet/llm_stats/llm_feedback.ex new file mode 100644 index 000000000..b940f8625 --- /dev/null +++ b/lib/cadet/llm_stats/llm_feedback.ex @@ -0,0 +1,34 @@ +defmodule Cadet.LLMStats.LLMFeedback do + @moduledoc """ + Schema for user feedback on the LLM "Generate Comments" feature. + """ + + use Ecto.Schema + import Ecto.Changeset + + schema "llm_feedback" do + belongs_to(:course, Cadet.Courses.Course) + belongs_to(:assessment, Cadet.Assessments.Assessment) + belongs_to(:question, Cadet.Assessments.Question) + belongs_to(:user, Cadet.Accounts.User) + + field(:rating, :integer) + field(:body, :string) + + timestamps() + end + + @required_fields ~w(course_id user_id body)a + @optional_fields ~w(assessment_id question_id rating)a + + def changeset(feedback, attrs) do + feedback + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) + |> validate_inclusion(:rating, 1..5) + |> foreign_key_constraint(:course_id) + |> foreign_key_constraint(:assessment_id) + |> foreign_key_constraint(:question_id) + |> foreign_key_constraint(:user_id) + end +end diff --git a/lib/cadet/llm_stats/llm_usage_log.ex b/lib/cadet/llm_stats/llm_usage_log.ex new file mode 100644 index 000000000..3fecc8080 --- /dev/null +++ b/lib/cadet/llm_stats/llm_usage_log.ex @@ -0,0 +1,33 @@ +defmodule Cadet.LLMStats.LLMUsageLog do + @moduledoc """ + Schema for logging each usage of the LLM "Generate Comments" feature. + """ + + use Ecto.Schema + import Ecto.Changeset + + schema "llm_usage_logs" do + belongs_to(:course, Cadet.Courses.Course) + belongs_to(:assessment, Cadet.Assessments.Assessment) + belongs_to(:question, Cadet.Assessments.Question) + belongs_to(:answer, Cadet.Assessments.Answer) + belongs_to(:submission, Cadet.Assessments.Submission) + belongs_to(:user, Cadet.Accounts.User) + + timestamps() + end + + @required_fields ~w(course_id assessment_id question_id answer_id submission_id user_id)a + + def changeset(log, attrs) do + log + |> cast(attrs, @required_fields) + |> validate_required(@required_fields) + |> foreign_key_constraint(:course_id) + |> foreign_key_constraint(:assessment_id) + |> foreign_key_constraint(:question_id) + |> foreign_key_constraint(:answer_id) + |> foreign_key_constraint(:submission_id) + |> foreign_key_constraint(:user_id) + end +end From 2f00d6b73d37206fea3e4c4f8b2263bcd2c8554f Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 3 Mar 2026 17:59:58 +0800 Subject: [PATCH 07/68] feat: add admin controller for LLM stats management --- .../admin_llm_stats_controller.ex | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex diff --git a/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex b/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex new file mode 100644 index 000000000..cd4ea140c --- /dev/null +++ b/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex @@ -0,0 +1,73 @@ +defmodule CadetWeb.AdminLLMStatsController do + @moduledoc """ + Controller for per-assessment and per-question LLM usage statistics and feedback. + """ + + use CadetWeb, :controller + require Logger + + alias Cadet.LLMStats + + @doc """ + GET /admin/llm-stats/:assessment_id + Returns assessment-level LLM usage statistics with per-question breakdown. + """ + def assessment_stats(conn, %{"course_id" => course_id, "assessment_id" => assessment_id}) do + stats = LLMStats.get_assessment_statistics(course_id, assessment_id) + json(conn, stats) + end + + @doc """ + GET /admin/llm-stats/:assessment_id/:question_id + Returns question-level LLM usage statistics. + """ + def question_stats(conn, %{ + "course_id" => course_id, + "assessment_id" => assessment_id, + "question_id" => question_id + }) do + stats = LLMStats.get_question_statistics(course_id, assessment_id, question_id) + json(conn, stats) + end + + @doc """ + GET /admin/llm-stats/:assessment_id/feedback + Returns feedback for an assessment, optionally filtered by question_id query param. + """ + def get_feedback(conn, %{"course_id" => course_id, "assessment_id" => assessment_id} = params) do + question_id = Map.get(params, "question_id") + feedback = LLMStats.get_feedback(course_id, assessment_id, question_id) + json(conn, feedback) + end + + @doc """ + POST /admin/llm-stats/:assessment_id/feedback + Submits new feedback for the LLM feature on an assessment (optionally for a specific question). + """ + def submit_feedback(conn, %{"course_id" => course_id, "assessment_id" => assessment_id} = params) do + user = conn.assigns[:current_user] + + attrs = %{ + course_id: course_id, + user_id: user.id, + assessment_id: assessment_id, + question_id: Map.get(params, "question_id"), + rating: Map.get(params, "rating"), + body: Map.get(params, "body") + } + + case LLMStats.submit_feedback(attrs) do + {:ok, _feedback} -> + conn + |> put_status(:created) + |> json(%{message: "Feedback submitted successfully"}) + + {:error, changeset} -> + Logger.error("Failed to submit LLM feedback: #{inspect(changeset.errors)}") + + conn + |> put_status(:bad_request) + |> json(%{error: "Failed to submit feedback"}) + end + end +end From 46686771b1960f476ac732f1b64838b7a18783b7 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 3 Mar 2026 18:00:35 +0800 Subject: [PATCH 08/68] feat: integrate LLM stats tracking into assessments --- lib/cadet/assessments/assessment.ex | 1 + lib/cadet/assessments/query.ex | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/cadet/assessments/assessment.ex b/lib/cadet/assessments/assessment.ex index edfce9bb6..472f95431 100644 --- a/lib/cadet/assessments/assessment.ex +++ b/lib/cadet/assessments/assessment.ex @@ -21,6 +21,7 @@ defmodule Cadet.Assessments.Assessment do field(:question_count, :integer, virtual: true) field(:graded_count, :integer, virtual: true) field(:is_grading_published, :boolean, virtual: true) + field(:has_llm_questions, :boolean, virtual: true, default: false) field(:title, :string) field(:is_published, :boolean, default: false) field(:summary_short, :string) diff --git a/lib/cadet/assessments/query.ex b/lib/cadet/assessments/query.ex index 85cca7468..a75612d45 100644 --- a/lib/cadet/assessments/query.ex +++ b/lib/cadet/assessments/query.ex @@ -20,7 +20,8 @@ defmodule Cadet.Assessments.Query do |> select([a, q], %Assessment{ a | max_xp: q.max_xp, - question_count: q.question_count + question_count: q.question_count, + has_llm_questions: q.has_llm_questions }) end @@ -49,7 +50,9 @@ defmodule Cadet.Assessments.Query do |> select([q], %{ assessment_id: q.assessment_id, max_xp: sum(q.max_xp), - question_count: count(q.id) + question_count: count(q.id), + has_llm_questions: + fragment("bool_or(? ->> 'llm_prompt' IS NOT NULL AND ? ->> 'llm_prompt' != '')", q.question, q.question) }) end end From ce5769d7945485aa925f992e6c41a17f69ace141 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 3 Mar 2026 18:01:28 +0800 Subject: [PATCH 09/68] feat: add LLM stats routes and views --- lib/cadet_web/admin_views/admin_assessments_view.ex | 3 ++- lib/cadet_web/router.ex | 6 ++++++ lib/cadet_web/views/assessments_view.ex | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/cadet_web/admin_views/admin_assessments_view.ex b/lib/cadet_web/admin_views/admin_assessments_view.ex index 00bc81849..f060722b0 100644 --- a/lib/cadet_web/admin_views/admin_assessments_view.ex +++ b/lib/cadet_web/admin_views/admin_assessments_view.ex @@ -32,7 +32,8 @@ defmodule CadetWeb.AdminAssessmentsView do maxTeamSize: :max_team_size, hasVotingFeatures: :has_voting_features, hasTokenCounter: :has_token_counter, - isVotingPublished: :is_voting_published + isVotingPublished: :is_voting_published, + isLlmGraded: &(&1.has_llm_questions || &1.llm_assessment_prompt != nil) }) end diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index e045dc637..f56a20689 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -244,6 +244,12 @@ defmodule CadetWeb.Router do :save_chosen_comments ) + # LLM Statistics & Feedback (per-assessment) + get("/llm-stats/:assessment_id", AdminLLMStatsController, :assessment_stats) + get("/llm-stats/:assessment_id/feedback", AdminLLMStatsController, :get_feedback) + post("/llm-stats/:assessment_id/feedback", AdminLLMStatsController, :submit_feedback) + get("/llm-stats/:assessment_id/:question_id", AdminLLMStatsController, :question_stats) + get("/users", AdminUserController, :index) get("/users/teamformation", AdminUserController, :get_students) put("/users", AdminUserController, :upsert_users_and_groups) diff --git a/lib/cadet_web/views/assessments_view.ex b/lib/cadet_web/views/assessments_view.ex index 3859b8615..343494949 100644 --- a/lib/cadet_web/views/assessments_view.ex +++ b/lib/cadet_web/views/assessments_view.ex @@ -35,7 +35,8 @@ defmodule CadetWeb.AssessmentsView do hasVotingFeatures: :has_voting_features, hasTokenCounter: :has_token_counter, isVotingPublished: :is_voting_published, - hoursBeforeEarlyXpDecay: & &1.config.hours_before_early_xp_decay + hoursBeforeEarlyXpDecay: & &1.config.hours_before_early_xp_decay, + isLlmGraded: &(&1.has_llm_questions || &1.llm_assessment_prompt != nil) }) end From aaf6a26fbcf0637ab5f6211db343b8f4989c52c1 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 3 Mar 2026 18:01:59 +0800 Subject: [PATCH 10/68] feat: implement AI comment generation controller --- .../controllers/generate_ai_comments.ex | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 68232f0eb..08611f5ba 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -4,7 +4,7 @@ defmodule CadetWeb.AICodeAnalysisController do require HTTPoison require Logger - alias Cadet.{Assessments, AIComments, Courses} + alias Cadet.{Assessments, AIComments, Courses, LLMStats} alias CadetWeb.{AICodeAnalysisController, AICommentsHelpers} # For logging outputs to both database and file @@ -104,7 +104,8 @@ defmodule CadetWeb.AICodeAnalysisController do llm_model: course.llm_model, llm_api_url: course.llm_api_url, course_prompt: course.llm_course_level_prompt, - assessment_prompt: Assessments.get_llm_assessment_prompt(answer.question_id) + assessment_prompt: Assessments.get_llm_assessment_prompt(answer.question_id), + course_id: course_id } ) else @@ -201,7 +202,8 @@ defmodule CadetWeb.AICodeAnalysisController do llm_model: llm_model, llm_api_url: llm_api_url, course_prompt: course_prompt, - assessment_prompt: assessment_prompt + assessment_prompt: assessment_prompt, + course_id: course_id } ) do # Combine prompts if llm_prompt exists @@ -236,6 +238,16 @@ defmodule CadetWeb.AICodeAnalysisController do content ) + # Log LLM usage for statistics + LLMStats.log_usage(%{ + course_id: course_id, + assessment_id: answer.question.assessment_id, + question_id: answer.question_id, + answer_id: answer.id, + submission_id: answer.submission_id, + user_id: conn.assigns.course_reg.user_id + }) + comments_list = String.split(content, "|||") filtered_comments = From 9a869f41e19ca2fc3da275161e5df01ae206c602 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:09:13 +0800 Subject: [PATCH 11/68] refactor(ai-comments): remove redundant final_comment endpoint and column --- lib/cadet/ai_comments.ex | 39 ++----- lib/cadet/ai_comments/ai_comment.ex | 3 +- .../controllers/generate_ai_comments.ex | 102 +----------------- lib/cadet_web/router.ex | 8 +- ...ove_final_comment_from_ai_comment_logs.exs | 9 ++ 5 files changed, 23 insertions(+), 138 deletions(-) create mode 100644 priv/repo/migrations/20260316120000_remove_final_comment_from_ai_comment_logs.exs diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 24cd68c79..0200ae505 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -40,24 +40,6 @@ defmodule Cadet.AIComments do ) end - @doc """ - Updates the final comment for a specific submission and question. - Returns the most recent comment entry for that submission/question. - """ - def update_final_comment(answer_id, final_comment) do - comment = get_latest_ai_comment(answer_id) - - case comment do - nil -> - {:error, :not_found} - - _ -> - comment - |> AIComment.changeset(%{final_comment: final_comment}) - |> Repo.update() - end - end - @doc """ Updates an existing AI comment with new attributes. """ @@ -98,7 +80,7 @@ defmodule Cadet.AIComments do Creates a new version entry for a specific comment index. Automatically determines the next version number. """ - def create_comment_version(ai_comment_id, comment_index, content, editor_id, diff \\ %{}) do + def create_comment_version(ai_comment_id, comment_index, content, editor_id) do next_version = Repo.one( from(v in AICommentVersion, @@ -108,18 +90,13 @@ defmodule Cadet.AIComments do ) + 1 %AICommentVersion{} - |> AICommentVersion.changeset( - Map.merge( - %{ - ai_comment_id: ai_comment_id, - comment_index: comment_index, - version_number: next_version, - content: content, - editor_id: editor_id - }, - diff - ) - ) + |> AICommentVersion.changeset(%{ + ai_comment_id: ai_comment_id, + comment_index: comment_index, + version_number: next_version, + content: content, + editor_id: editor_id + }) |> Repo.insert() end diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex index e986f739b..bcd250378 100644 --- a/lib/cadet/ai_comments/ai_comment.ex +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -11,7 +11,6 @@ defmodule Cadet.AIComments.AIComment do field(:answers_json, :string) field(:response, :string) field(:error, :string) - field(:final_comment, :string) field(:selected_indices, {:array, :integer}) field(:finalized_at, :utc_datetime) @@ -24,7 +23,7 @@ defmodule Cadet.AIComments.AIComment do end @required_fields ~w(answer_id raw_prompt answers_json)a - @optional_fields ~w(response error final_comment selected_indices finalized_by_id finalized_at)a + @optional_fields ~w(response error selected_indices finalized_by_id finalized_at)a def changeset(ai_comment, attrs) do ai_comment diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 08611f5ba..5fb2d3894 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -32,10 +32,7 @@ defmodule CadetWeb.AICodeAnalysisController do end existing_comment -> - # Convert the existing comment struct to a map before merging - updated_attrs = Map.merge(Map.from_struct(existing_comment), attrs) - - case AIComments.update_ai_comment(existing_comment.id, updated_attrs) do + case AIComments.update_ai_comment(existing_comment.id, attrs) do {:error, :not_found} -> Logger.error("AI comment to update not found in database") {:error, :not_found} @@ -285,24 +282,6 @@ defmodule CadetWeb.AICodeAnalysisController do end end - @doc """ - Saves the final comment chosen for a submission. - """ - def save_final_comment(conn, %{ - "answer_id" => answer_id, - "comment" => comment - }) do - case AIComments.update_final_comment(answer_id, comment) do - {:ok, _updated_comment} -> - json(conn, %{"status" => "success"}) - - {:error, changeset} -> - conn - |> put_status(:unprocessable_entity) - |> text("Failed to save final comment") - end - end - @doc """ Saves the chosen comment indices and optional edits for each selected comment. Expects: selected_indices (list of ints), edits (optional map of index => edited_text). @@ -310,8 +289,6 @@ defmodule CadetWeb.AICodeAnalysisController do def save_chosen_comments( conn, params = %{ - "submissionid" => _submission_id, - "questionid" => _question_id, "answer_id" => answer_id, "selected_indices" => selected_indices } @@ -322,26 +299,16 @@ defmodule CadetWeb.AICodeAnalysisController do with ai_comment when not is_nil(ai_comment) <- AIComments.get_latest_ai_comment(answer_id), {:ok, _updated} <- AIComments.save_selected_comments(answer_id, selected_indices, editor_id) do - # Split the original response into individual comments - original_comments = - (ai_comment.response || "") - |> String.split("|||") - |> Enum.map(&String.trim/1) - |> Enum.filter(&(&1 != "")) - # Create version entries for each edit version_results = Enum.map(edits, fn {index_str, edited_text} -> index = String.to_integer(index_str) - original = Enum.at(original_comments, index, "") - diff = compute_diff(original, edited_text) AIComments.create_comment_version( ai_comment.id, index, edited_text, - editor_id, - diff + editor_id ) end) @@ -367,36 +334,6 @@ defmodule CadetWeb.AICodeAnalysisController do end end - defp compute_diff(original, edited) do - original_tokens = tokenize(original) - edited_tokens = tokenize(edited) - - diff = List.myers_difference(original_tokens, edited_tokens) - - ops = - diff - |> Enum.flat_map(fn - {:eq, tokens} -> [%{op: "eq", text: Enum.join(tokens)}] - {:del, tokens} -> [%{op: "del", text: Enum.join(tokens)}] - {:ins, tokens} -> [%{op: "add", text: Enum.join(tokens)}] - end) - - unified = - Enum.map_join(ops, fn - %{op: "eq", text: text} -> " " <> text - %{op: "del", text: text} -> "-" <> text - %{op: "add", text: text} -> "+" <> text - end) - - %{diff_json: %{"ops" => ops}, diff_unified: unified} - end - - # Splits text into tokens at word boundaries, preserving whitespace and punctuation - # as separate tokens so the diff is word-level accurate. - defp tokenize(text) do - Regex.split(~r/\b/, text, include_captures: false, trim: true) - end - swagger_path :generate_ai_comments do post("/courses/{course_id}/admin/generate-comments/{answer_id}") @@ -419,30 +356,8 @@ defmodule CadetWeb.AICodeAnalysisController do response(403, "LLM grading is not enabled for this course") end - swagger_path :save_final_comment do - post("/courses/{course_id}/admin/save-final-comment/{answer_id}") - - summary("Save the final comment chosen for a submission.") - - security([%{JWT: []}]) - - consumes("application/json") - produces("application/json") - - parameters do - course_id(:path, :integer, "course id", required: true) - answer_id(:path, :integer, "answer id", required: true) - comment(:body, :string, "The final comment to save", required: true) - end - - response(200, "OK", Schema.ref(:SaveFinalComment)) - response(400, "Invalid or missing parameter(s)") - response(401, "Unauthorized") - response(403, "Forbidden") - end - swagger_path :save_chosen_comments do - post("/courses/{course_id}/admin/save-chosen-comments/{submissionid}/{questionid}") + post("/courses/{course_id}/admin/save-chosen-comments/{answer_id}") summary("Save chosen comment indices and optional edits for a submission.") @@ -453,8 +368,7 @@ defmodule CadetWeb.AICodeAnalysisController do parameters do course_id(:path, :integer, "course id", required: true) - submissionid(:path, :integer, "submission id", required: true) - questionid(:path, :integer, "question id", required: true) + answer_id(:path, :integer, "answer id", required: true) body(:body, Schema.ref(:SaveChosenCommentsBody), "Chosen comments payload", required: true) end @@ -472,17 +386,9 @@ defmodule CadetWeb.AICodeAnalysisController do comments(:string, "AI-generated comments on the submission answers") end end, - SaveFinalComment: - swagger_schema do - properties do - status(:string, "Status of the operation") - end - end, SaveChosenCommentsBody: swagger_schema do properties do - answer_id(:integer, "The answer ID", required: true) - selected_indices(Schema.ref(:IntegerArray), "Indices of chosen comments", required: true ) diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index f56a20689..cdbe0b871 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -233,13 +233,7 @@ defmodule CadetWeb.Router do ) post( - "/save-final-comment/:answer_id", - AICodeAnalysisController, - :save_final_comment - ) - - post( - "/save-chosen-comments/:submissionid/:questionid", + "/save-chosen-comments/:answer_id", AICodeAnalysisController, :save_chosen_comments ) diff --git a/priv/repo/migrations/20260316120000_remove_final_comment_from_ai_comment_logs.exs b/priv/repo/migrations/20260316120000_remove_final_comment_from_ai_comment_logs.exs new file mode 100644 index 000000000..a4c3f44e7 --- /dev/null +++ b/priv/repo/migrations/20260316120000_remove_final_comment_from_ai_comment_logs.exs @@ -0,0 +1,9 @@ +defmodule Cadet.Repo.Migrations.RemoveFinalCommentFromAiCommentLogs do + use Ecto.Migration + + def change do + alter table(:ai_comment_logs) do + remove(:final_comment) + end + end +end From 7fc9ba642348f3a1d408906423f3800f7a5ff26b Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:09:33 +0800 Subject: [PATCH 12/68] refactor(ai-comments): drop myers diff generation and diff columns from versions --- lib/cadet/ai_comments/ai_comment_version.ex | 4 +--- ...00_remove_diff_columns_from_ai_comment_versions.exs | 10 ++++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 priv/repo/migrations/20260316123000_remove_diff_columns_from_ai_comment_versions.exs diff --git a/lib/cadet/ai_comments/ai_comment_version.ex b/lib/cadet/ai_comments/ai_comment_version.ex index c120c68c3..2de1b88fa 100644 --- a/lib/cadet/ai_comments/ai_comment_version.ex +++ b/lib/cadet/ai_comments/ai_comment_version.ex @@ -11,8 +11,6 @@ defmodule Cadet.AIComments.AICommentVersion do field(:comment_index, :integer) field(:version_number, :integer) field(:content, :string) - field(:diff_json, :map) - field(:diff_unified, :string) belongs_to(:ai_comment, Cadet.AIComments.AIComment) belongs_to(:editor, Cadet.Accounts.User, foreign_key: :editor_id) @@ -21,7 +19,7 @@ defmodule Cadet.AIComments.AICommentVersion do end @required_fields ~w(ai_comment_id comment_index version_number content)a - @optional_fields ~w(editor_id diff_json diff_unified)a + @optional_fields ~w(editor_id)a def changeset(version, attrs) do version diff --git a/priv/repo/migrations/20260316123000_remove_diff_columns_from_ai_comment_versions.exs b/priv/repo/migrations/20260316123000_remove_diff_columns_from_ai_comment_versions.exs new file mode 100644 index 000000000..2773886e7 --- /dev/null +++ b/priv/repo/migrations/20260316123000_remove_diff_columns_from_ai_comment_versions.exs @@ -0,0 +1,10 @@ +defmodule Cadet.Repo.Migrations.RemoveDiffColumnsFromAiCommentVersions do + use Ecto.Migration + + def change do + alter table(:ai_comment_versions) do + remove(:diff_json) + remove(:diff_unified) + end + end +end From 1274299cfb7083d486e48ebf2c8da2e7809a5d90 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 10:37:25 +0800 Subject: [PATCH 13/68] Stop tracking config/cadet.exs --- .gitignore | 2 + config/cadet.exs | 179 ----------------------------------------------- 2 files changed, 2 insertions(+), 179 deletions(-) delete mode 100644 config/cadet.exs diff --git a/.gitignore b/.gitignore index 3ab2b5e8c..45a99d0d2 100644 --- a/.gitignore +++ b/.gitignore @@ -94,6 +94,8 @@ erl_crash.dump # variables. /config/*secrets.exs +/config/cadet.exs + # Uploaded file /cs1101s /uploads diff --git a/config/cadet.exs b/config/cadet.exs deleted file mode 100644 index f20aae580..000000000 --- a/config/cadet.exs +++ /dev/null @@ -1,179 +0,0 @@ -# Example production configuration file - -import Config - -# See https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-runtime-configuration -# except for cors_endpoints, load_from_system_env which are custom -config :cadet, CadetWeb.Endpoint, - # See https://hexdocs.pm/corsica/Corsica.html#module-origins - # Remove for "*" - cors_endpoints: "example.com", - server: true, - # If true, expects an environment variable PORT specifying the port to listen on - load_from_system_env: true, - url: [host: "api.example.com", port: 80], - # You can specify the port here instead - # e.g http: [compress: true, port: 4000] - http: [compress: true], - # Generate using `mix phx.gen.secret` - secret_key_base: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" - -config :cadet, Cadet.Auth.Guardian, - issuer: "cadet", - # Generate using `mix phx.gen.secret` - secret_key: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" - -config :cadet, Cadet.Repo, - # Do not change this, only Postgres is supported - # (This is here because of how configuration works in Elixir.) - adapter: Ecto.Adapters.Postgres, - # The AWS Secrets Manager secret name containing the database connection details - rds_secret_name: "-cadet-db", - # Alternatively, you can include the credentials here: - # (comment out or remove rds_secret_name) - # username: "postgres", - # password: "postgres", - # database: "cadet_stg", - # hostname: "localhost", - pool_size: 10 - -config :cadet, - identity_providers: %{ - # # To use authentication with ADFS. - # "nusnet" => - # {Cadet.Auth.Providers.ADFS, - # %{ - # # The OAuth2 token endpoint. - # token_endpoint: "https://my-adfs/adfs/oauth2/token" - # }}, - # # An example of OpenID authentication with Cognito. Any OpenID-compliant - # # provider should work. - # "cognito" => - # {Cadet.Auth.Providers.OpenID, - # %{ - # # This should match a key in openid_connect_providers below - # openid_provider: :cognito, - # # You may need to write your own claim extractor for other providers - # claim_extractor: Cadet.Auth.Providers.CognitoClaimExtractor - # }}, - - # # Example SAML authentication with NUS Student IdP - # "test_saml" => - # {Cadet.Auth.Providers.SAML, - # %{ - # assertion_extractor: Cadet.Auth.Providers.NusstuAssertionExtractor, - # client_redirect_url: "http://cadet.frontend:8000/login/callback" - # }}, - - "test" => - {Cadet.Auth.Providers.Config, - [ - %{ - token: "admin_token", - code: "admin_code", - name: "Test Admin", - username: "admin", - role: :admin - }, - %{ - token: "staff_token", - code: "staff_code", - name: "Test Staff", - username: "staff", - role: :staff - }, - %{ - token: "student_token", - code: "student_code", - name: "Test Student", - username: "student", - role: :student - } - ]} - }, - # See https://hexdocs.pm/openid_connect/readme.html - # openid_connect_providers: [ - # cognito: [ - # discovery_document_uri: "", - # client_id: "", - # client_secret: "", - # response_type: "code", - # scope: "openid profile" - # ] - # ], - autograder: [ - lambda_name: "-grader" - ], - uploader: [ - assets_bucket: "-assets", - assets_prefix: "courses/", - sourcecasts_bucket: "-cadet-sourcecasts" - ], - # Configuration for Sling integration (executing on remote devices) - remote_execution: [ - # Prefix for AWS IoT thing names - thing_prefix: "-sling", - # AWS IoT thing group to put created things into (must be set-up beforehand) - thing_group: "-sling", - # Role ARN to use when generating signed client MQTT-WS URLs (must be set-up beforehand) - # Note, you need to specify the correct account ID. Find it in AWS IAM at the bottom left. - client_role_arn: "arn:aws:iam:::role/-cadet-frontend" - ] - -# Sentry DSN. This is only really useful to the NUS SoC deployments, but you can -# specify a DSN here if you wish to track backend errors. -# config :sentry, -# dsn: "https://public_key/sentry.io/somethingsomething" - -# If you are not running on EC2, you will need to configure an AWS access token -# for the backend to access AWS resources: -# -# This will make ExAWS read the values from the corresponding environment variables. -# config :ex_aws, -# access_key_id: [{:system, "AWS_ACCESS_KEY_ID"}], -# secret_access_key: [{:system, "AWS_SECRET_ACCESS_KEY"}] -# -# You can also just specify the values directly: -# config :ex_aws, -# access_key_id: "ACCESS KEY", -# secret_access_key: "SECRET KEY" -# -# You may also want to change the AWS or S3 region used for all resources: -# (Note, the default is ap-southeast-1 i.e. Singapore) -# config :ex_aws, -# region: "ap-southeast-1", -# s3: [ -# scheme: "https://", -# host: "s3.ap-southeast-1.amazonaws.com", -# region: "ap-southeast-1" -# ] - -# You may also want to change the timezone used for scheduled jobs -# config :cadet, Cadet.Jobs.Scheduler, -# timezone: "Asia/Singapore", - -# # Additional configuration for SAML authentication -# # For more details, see https://github.com/handnot2/samly -# config :samly, Samly.Provider, -# idp_id_from: :path_segment, -# service_providers: [ -# %{ -# id: "source-academy-backend", -# certfile: "example_path/certfile.pem", -# keyfile: "example_path/keyfile.pem" -# } -# ], -# identity_providers: [ -# %{ -# id: "student", -# sp_id: "source-academy-backend", -# base_url: "https://example_backend/sso", -# metadata_file: "student_idp_metadata.xml" -# }, -# %{ -# id: "staff", -# sp_id: "source-academy-backend", -# base_url: "https://example_backend/sso", -# metadata_file: "staff_idp_metadata.xml" -# } -# ] From a999538cdb31cd716118c080cff7a71a4972159a Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 10:43:25 +0800 Subject: [PATCH 14/68] fix: Potential crash if a non-numeric key is provided --- .../controllers/generate_ai_comments.ex | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 5fb2d3894..af4323cb7 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -298,12 +298,11 @@ defmodule CadetWeb.AICodeAnalysisController do with ai_comment when not is_nil(ai_comment) <- AIComments.get_latest_ai_comment(answer_id), {:ok, _updated} <- - AIComments.save_selected_comments(answer_id, selected_indices, editor_id) do + AIComments.save_selected_comments(answer_id, selected_indices, editor_id), + {:ok, parsed_edits} <- parse_edits(edits) do # Create version entries for each edit version_results = - Enum.map(edits, fn {index_str, edited_text} -> - index = String.to_integer(index_str) - + Enum.map(parsed_edits, fn {index, edited_text} -> AIComments.create_comment_version( ai_comment.id, index, @@ -327,6 +326,11 @@ defmodule CadetWeb.AICodeAnalysisController do |> put_status(:not_found) |> text("AI comment not found for this answer") + {:error, :invalid_edits} -> + conn + |> put_status(:unprocessable_entity) + |> text("Invalid edits payload") + {:error, _} -> conn |> put_status(:unprocessable_entity) @@ -334,6 +338,34 @@ defmodule CadetWeb.AICodeAnalysisController do end end + defp parse_edits(edits) when is_map(edits) do + edits + |> Enum.reduce_while({:ok, []}, fn {index_str, edited_text}, {:ok, acc} -> + case {parse_edit_index(index_str), edited_text} do + {{:ok, index}, edited_text} when is_binary(edited_text) -> + {:cont, {:ok, [{index, edited_text} | acc]}} + + _ -> + {:halt, {:error, :invalid_edits}} + end + end) + |> case do + {:ok, parsed_edits} -> {:ok, Enum.reverse(parsed_edits)} + {:error, :invalid_edits} -> {:error, :invalid_edits} + end + end + + defp parse_edits(_), do: {:error, :invalid_edits} + + defp parse_edit_index(index_str) when is_binary(index_str) do + case Integer.parse(index_str) do + {index, ""} -> {:ok, index} + _ -> {:error, :invalid_edits} + end + end + + defp parse_edit_index(_), do: {:error, :invalid_edits} + swagger_path :generate_ai_comments do post("/courses/{course_id}/admin/generate-comments/{answer_id}") From 3529738c9e95aaef35659bf16e23e60afdce7b9e Mon Sep 17 00:00:00 2001 From: tzj04 Date: Tue, 17 Mar 2026 11:02:43 +0800 Subject: [PATCH 15/68] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- ...20260316120000_remove_final_comment_from_ai_comment_logs.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/repo/migrations/20260316120000_remove_final_comment_from_ai_comment_logs.exs b/priv/repo/migrations/20260316120000_remove_final_comment_from_ai_comment_logs.exs index a4c3f44e7..ce3e17419 100644 --- a/priv/repo/migrations/20260316120000_remove_final_comment_from_ai_comment_logs.exs +++ b/priv/repo/migrations/20260316120000_remove_final_comment_from_ai_comment_logs.exs @@ -3,7 +3,7 @@ defmodule Cadet.Repo.Migrations.RemoveFinalCommentFromAiCommentLogs do def change do alter table(:ai_comment_logs) do - remove(:final_comment) + remove(:final_comment, :text) end end end From d253c61af6fd3aab8574b53d54fd648b6a38c946 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 13:48:31 +0800 Subject: [PATCH 16/68] fix: persist and restore AI comment selections correctly --- .../admin_views/admin_grading_view.ex | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index f6a90d17d..69a46f343 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -1,6 +1,7 @@ defmodule CadetWeb.AdminGradingView do use CadetWeb, :view + alias Cadet.AIComments import CadetWeb.AssessmentsHelpers alias CadetWeb.AICodeAnalysisController @@ -172,14 +173,34 @@ defmodule CadetWeb.AdminGradingView do end defp extract_ai_comments_per_answer(id, ai_comments) do - matching_comment = + latest_comment = ai_comments - # Equivalent to fn comment -> comment.question_id == question_id end - |> Enum.find(&(&1.answer_id == id)) + |> Enum.filter(&(&1.answer_id == id)) + |> case do + [] -> nil + comments -> Enum.max_by(comments, & &1.inserted_at, NaiveDateTime) + end - case matching_comment do + case latest_comment do nil -> nil - comment -> %{response: comment.response, insertedAt: comment.inserted_at} + comment -> + selected_indices = comment.selected_indices || [] + + selected_edits = + selected_indices + |> Enum.reduce(%{}, fn index, acc -> + case AIComments.get_latest_version(comment.id, index) do + nil -> acc + version -> Map.put(acc, index, version.content) + end + end) + + %{ + response: comment.response, + insertedAt: comment.inserted_at, + selectedIndices: selected_indices, + selectedEdits: selected_edits + } end end From d279faa763357ae6a9491307574b520c77ab640c Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 14:21:20 +0800 Subject: [PATCH 17/68] fix: prevent race condition in create_comment_version using advisory lock --- lib/cadet/ai_comments.ex | 48 ++++++++++++++++++++----------- test/cadet/ai_comments_test.exs | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 16 deletions(-) create mode 100644 test/cadet/ai_comments_test.exs diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 0200ae505..9b444ff11 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -81,23 +81,39 @@ defmodule Cadet.AIComments do Automatically determines the next version number. """ def create_comment_version(ai_comment_id, comment_index, content, editor_id) do - next_version = - Repo.one( - from(v in AICommentVersion, - where: v.ai_comment_id == ^ai_comment_id and v.comment_index == ^comment_index, - select: coalesce(max(v.version_number), 0) - ) - ) + 1 + Repo.transaction(fn -> + # Serialize version creation per (ai_comment_id, comment_index) to avoid duplicate version numbers. + case Repo.query("SELECT pg_advisory_xact_lock($1, $2)", [ai_comment_id, comment_index]) do + {:ok, _} -> + next_version = + Repo.one( + from(v in AICommentVersion, + where: v.ai_comment_id == ^ai_comment_id and v.comment_index == ^comment_index, + select: coalesce(max(v.version_number), 0) + ) + ) + 1 - %AICommentVersion{} - |> AICommentVersion.changeset(%{ - ai_comment_id: ai_comment_id, - comment_index: comment_index, - version_number: next_version, - content: content, - editor_id: editor_id - }) - |> Repo.insert() + case %AICommentVersion{} + |> AICommentVersion.changeset(%{ + ai_comment_id: ai_comment_id, + comment_index: comment_index, + version_number: next_version, + content: content, + editor_id: editor_id + }) + |> Repo.insert() do + {:ok, version} -> version + {:error, changeset} -> Repo.rollback(changeset) + end + + {:error, error} -> + Repo.rollback(error) + end + end) + |> case do + {:ok, version} -> {:ok, version} + {:error, reason} -> {:error, reason} + end end @doc """ diff --git a/test/cadet/ai_comments_test.exs b/test/cadet/ai_comments_test.exs new file mode 100644 index 000000000..8c9120fb0 --- /dev/null +++ b/test/cadet/ai_comments_test.exs @@ -0,0 +1,51 @@ +defmodule Cadet.AICommentsTest do + use Cadet.DataCase + + alias Cadet.{AIComments, Repo} + alias Cadet.AIComments.AICommentVersion + alias Ecto.Adapters.SQL.Sandbox + + setup do + course = insert(:course) + assessment = insert(:assessment, course: course) + submission = insert(:submission, assessment: assessment) + question = insert(:programming_question, assessment: assessment) + answer = insert(:answer, submission: submission, question: question) + editor = insert(:user) + + {:ok, ai_comment} = + AIComments.create_ai_comment(%{ + answer_id: answer.id, + raw_prompt: "prompt", + answers_json: "[]" + }) + + {:ok, ai_comment: ai_comment, editor: editor} + end + + test "creates distinct version numbers for concurrent edits", %{ai_comment: ai_comment, editor: editor} do + parent = self() + + create_version = fn content -> + Sandbox.allow(Repo, parent, self()) + AIComments.create_comment_version(ai_comment.id, 0, content, editor.id) + end + + task_1 = Task.async(fn -> create_version.("first edit") end) + task_2 = Task.async(fn -> create_version.("second edit") end) + + assert {:ok, _} = Task.await(task_1, 5_000) + assert {:ok, _} = Task.await(task_2, 5_000) + + versions = + Repo.all( + from(v in AICommentVersion, + where: v.ai_comment_id == ^ai_comment.id and v.comment_index == 0, + order_by: [asc: v.version_number] + ) + ) + + assert Enum.map(versions, & &1.version_number) == [1, 2] + assert Enum.sort(Enum.map(versions, & &1.content)) == ["first edit", "second edit"] + end +end From 284939d10d7a62f5d488cc52f38c382bf3c62956 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 14:31:10 +0800 Subject: [PATCH 18/68] test: update assessment controller expectations for isLlmGraded field --- .../admin_assessments_controller_test.exs | 6 ++++-- .../controllers/assessments_controller_test.exs | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/cadet_web/admin_controllers/admin_assessments_controller_test.exs b/test/cadet_web/admin_controllers/admin_assessments_controller_test.exs index b1cfbde5a..ee76ac374 100644 --- a/test/cadet_web/admin_controllers/admin_assessments_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_assessments_controller_test.exs @@ -94,7 +94,8 @@ defmodule CadetWeb.AdminAssessmentsControllerTest do "earlySubmissionXp" => &1.config.early_submission_xp, "hasVotingFeatures" => &1.has_voting_features, "hasTokenCounter" => &1.has_token_counter, - "isVotingPublished" => false + "isVotingPublished" => false, + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt != nil, } ) @@ -145,7 +146,8 @@ defmodule CadetWeb.AdminAssessmentsControllerTest do "earlySubmissionXp" => &1.config.early_submission_xp, "hasVotingFeatures" => &1.has_voting_features, "hasTokenCounter" => &1.has_token_counter, - "isVotingPublished" => false + "isVotingPublished" => false, + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt != nil, } ) diff --git a/test/cadet_web/controllers/assessments_controller_test.exs b/test/cadet_web/controllers/assessments_controller_test.exs index 17e7cca72..cb4698cfe 100644 --- a/test/cadet_web/controllers/assessments_controller_test.exs +++ b/test/cadet_web/controllers/assessments_controller_test.exs @@ -84,7 +84,8 @@ defmodule CadetWeb.AssessmentsControllerTest do "hasVotingFeatures" => &1.has_voting_features, "hasTokenCounter" => &1.has_token_counter, "isVotingPublished" => false, - "hoursBeforeEarlyXpDecay" => &1.config.hours_before_early_xp_decay + "hoursBeforeEarlyXpDecay" => &1.config.hours_before_early_xp_decay, + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt != nil } ) @@ -175,7 +176,8 @@ defmodule CadetWeb.AssessmentsControllerTest do "hasVotingFeatures" => &1.has_voting_features, "hasTokenCounter" => &1.has_token_counter, "isVotingPublished" => false, - "hoursBeforeEarlyXpDecay" => &1.config.hours_before_early_xp_decay + "hoursBeforeEarlyXpDecay" => &1.config.hours_before_early_xp_decay, + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt != nil, } ) @@ -297,7 +299,8 @@ defmodule CadetWeb.AssessmentsControllerTest do false else &1.is_published - end + end, + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt != nil } ) From 4ab4e788962d65e65191e9e6062e3a408de99c52 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 17:34:59 +0800 Subject: [PATCH 19/68] refactor(migrations): fold ai_comment_versions diff-column removal into create migration --- .../20260225120000_create_ai_comment_versions.exs | 2 -- ...00_remove_diff_columns_from_ai_comment_versions.exs | 10 ---------- 2 files changed, 12 deletions(-) delete mode 100644 priv/repo/migrations/20260316123000_remove_diff_columns_from_ai_comment_versions.exs diff --git a/priv/repo/migrations/20260225120000_create_ai_comment_versions.exs b/priv/repo/migrations/20260225120000_create_ai_comment_versions.exs index 04f70e480..f930f05e6 100644 --- a/priv/repo/migrations/20260225120000_create_ai_comment_versions.exs +++ b/priv/repo/migrations/20260225120000_create_ai_comment_versions.exs @@ -8,8 +8,6 @@ defmodule Cadet.Repo.Migrations.CreateAiCommentVersions do add(:version_number, :integer, null: false, default: 1) add(:editor_id, references(:users, on_delete: :nilify_all)) add(:content, :text) - add(:diff_json, :map) - add(:diff_unified, :text) timestamps() end diff --git a/priv/repo/migrations/20260316123000_remove_diff_columns_from_ai_comment_versions.exs b/priv/repo/migrations/20260316123000_remove_diff_columns_from_ai_comment_versions.exs deleted file mode 100644 index 2773886e7..000000000 --- a/priv/repo/migrations/20260316123000_remove_diff_columns_from_ai_comment_versions.exs +++ /dev/null @@ -1,10 +0,0 @@ -defmodule Cadet.Repo.Migrations.RemoveDiffColumnsFromAiCommentVersions do - use Ecto.Migration - - def change do - alter table(:ai_comment_versions) do - remove(:diff_json) - remove(:diff_unified) - end - end -end From fd9f6156b116ded30c3477101d413ad223c29016 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 19:49:26 +0800 Subject: [PATCH 20/68] fix: swagger generation --- lib/cadet_web/controllers/generate_ai_comments.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index af4323cb7..e58336028 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -421,7 +421,7 @@ defmodule CadetWeb.AICodeAnalysisController do SaveChosenCommentsBody: swagger_schema do properties do - selected_indices(Schema.ref(:IntegerArray), "Indices of chosen comments", + selected_indices(Schema.array(:integer), "Indices of chosen comments", required: true ) From d5a513be993fd04ea6aeb64a423e99985347c4a8 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 19:54:55 +0800 Subject: [PATCH 21/68] Validate and safely parse answer_id in save_chosen_comments to prevent 500 on invalid route params --- .../controllers/generate_ai_comments.ex | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index e58336028..7c8ac9254 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -292,13 +292,15 @@ defmodule CadetWeb.AICodeAnalysisController do "answer_id" => answer_id, "selected_indices" => selected_indices } - ) do + ) + when is_ecto_id(answer_id) do editor_id = conn.assigns.course_reg.user_id edits = Map.get(params, "edits", %{}) - with ai_comment when not is_nil(ai_comment) <- AIComments.get_latest_ai_comment(answer_id), + with {:ok, answer_id_parsed} <- parse_answer_id(answer_id), + ai_comment when not is_nil(ai_comment) <- AIComments.get_latest_ai_comment(answer_id_parsed), {:ok, _updated} <- - AIComments.save_selected_comments(answer_id, selected_indices, editor_id), + AIComments.save_selected_comments(answer_id_parsed, selected_indices, editor_id), {:ok, parsed_edits} <- parse_edits(edits) do # Create version entries for each edit version_results = @@ -321,6 +323,11 @@ defmodule CadetWeb.AICodeAnalysisController do |> text("Failed to save some comment versions") end else + {:error, :invalid_answer_id} -> + conn + |> put_status(:bad_request) + |> text("Invalid answer ID format") + nil -> conn |> put_status(:not_found) @@ -338,6 +345,17 @@ defmodule CadetWeb.AICodeAnalysisController do end end + defp parse_answer_id(answer_id) when is_integer(answer_id), do: {:ok, answer_id} + + defp parse_answer_id(answer_id) when is_binary(answer_id) do + case Integer.parse(answer_id) do + {parsed, ""} -> {:ok, parsed} + _ -> {:error, :invalid_answer_id} + end + end + + defp parse_answer_id(_), do: {:error, :invalid_answer_id} + defp parse_edits(edits) when is_map(edits) do edits |> Enum.reduce_while({:ok, []}, fn {index_str, edited_text}, {:ok, acc} -> From c102625b1fa2af87fa2661859a68b03c54547213 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 22:32:41 +0800 Subject: [PATCH 22/68] refactor(deps): migrate arc/arc_ecto to waffle/waffle_ecto Replace abandoned Arc libraries with actively maintained Waffle forks. - swap deps: arc -> waffle (~> 1.1), arc_ecto -> waffle_ecto (~> 0.0.12) - rename Arc modules/usages to Waffle in lib/ - rename :arc config/atoms to :waffle in config/ - refresh deps and lockfile - pin ex_aws_s3 to 2.4.0 to preserve existing test behavior Closes source-academy/backend#1269 --- config/config.exs | 6 +++--- config/test.exs | 2 +- lib/cadet.ex | 4 ++-- lib/cadet/assessments/assessment.ex | 2 +- lib/cadet/courses/sourcecast.ex | 2 +- lib/cadet/courses/sourcecast_upload.ex | 4 ++-- mix.exs | 6 +++--- mix.lock | 4 ++-- test/support/data_case.ex | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/config/config.exs b/config/config.exs index 5799f7060..86a789c38 100644 --- a/config/config.exs +++ b/config/config.exs @@ -63,10 +63,10 @@ config :ex_aws, config :ex_aws, :hackney_opts, recv_timeout: 660_000 -# Configure Arc File Upload -config :arc, virtual_host: true +# Configure Waffle File Upload +config :waffle, virtual_host: true # Or uncomment below to use local storage -# config :arc, storage: Arc.Storage.Local +# config :waffle, storage: Waffle.Storage.Local # Configures Sentry config :sentry, diff --git a/config/test.exs b/config/test.exs index 7e9aa8f86..04ef581a1 100644 --- a/config/test.exs +++ b/config/test.exs @@ -90,7 +90,7 @@ config :cadet, client_role_arn: "test" ] -config :arc, storage: Arc.Storage.Local +config :waffle, storage: Waffle.Storage.Local if "test.secrets.exs" |> Path.expand(__DIR__) |> File.exists?(), do: import_config("test.secrets.exs") diff --git a/lib/cadet.ex b/lib/cadet.ex index cc451955a..b797511e5 100644 --- a/lib/cadet.ex +++ b/lib/cadet.ex @@ -40,8 +40,8 @@ defmodule Cadet do def remote_assets do quote do - use Arc.Definition - use Arc.Ecto.Definition + use Waffle.Definition + use Waffle.Ecto.Definition end end end diff --git a/lib/cadet/assessments/assessment.ex b/lib/cadet/assessments/assessment.ex index 472f95431..0b1cbdac2 100644 --- a/lib/cadet/assessments/assessment.ex +++ b/lib/cadet/assessments/assessment.ex @@ -4,7 +4,7 @@ defmodule Cadet.Assessments.Assessment do (mission, sidequest, path, and contest) """ use Cadet, :model - use Arc.Ecto.Schema + use Waffle.Ecto.Schema alias Cadet.Repo alias Cadet.Assessments.{AssessmentAccess, Question, SubmissionStatus, Upload} diff --git a/lib/cadet/courses/sourcecast.ex b/lib/cadet/courses/sourcecast.ex index 330663ab8..19683bb14 100644 --- a/lib/cadet/courses/sourcecast.ex +++ b/lib/cadet/courses/sourcecast.ex @@ -3,7 +3,7 @@ defmodule Cadet.Courses.Sourcecast do Sourcecast stores audio files and deltas for playback """ use Cadet, :model - use Arc.Ecto.Schema + use Waffle.Ecto.Schema alias Cadet.Accounts.User alias Cadet.Courses.{Course, SourcecastUpload} diff --git a/lib/cadet/courses/sourcecast_upload.ex b/lib/cadet/courses/sourcecast_upload.ex index 1d9a57c78..0b75130a7 100644 --- a/lib/cadet/courses/sourcecast_upload.ex +++ b/lib/cadet/courses/sourcecast_upload.ex @@ -2,8 +2,8 @@ defmodule Cadet.Courses.SourcecastUpload do @moduledoc """ Represents an uploaded file for Sourcecast """ - use Arc.Definition - use Arc.Ecto.Definition + use Waffle.Definition + use Waffle.Ecto.Definition @extension_whitelist ~w(.wav) @versions [:original] diff --git a/mix.exs b/mix.exs index 79dcf4c4a..038a72637 100644 --- a/mix.exs +++ b/mix.exs @@ -57,14 +57,14 @@ defmodule Cadet.Mixfile do # Type `mix help deps` for examples and options. defp deps do [ - {:arc, "~> 0.11"}, - {:arc_ecto, "~> 0.11"}, + {:waffle, "~> 1.1"}, + {:waffle_ecto, "~> 0.0.12"}, {:corsica, "~> 2.1"}, {:csv, "~> 3.2"}, {:ecto_enum, "~> 1.0"}, {:ex_aws, "~> 2.1", override: true}, {:ex_aws_lambda, "~> 2.0"}, - {:ex_aws_s3, "~> 2.0"}, + {:ex_aws_s3, "2.4.0"}, {:ex_aws_secretsmanager, "~> 2.0"}, {:ex_aws_sts, "~> 2.1"}, {:ex_json_schema, "~> 0.11.0"}, diff --git a/mix.lock b/mix.lock index 8c286b55f..813c953d2 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,4 @@ %{ - "arc": {:hex, :arc, "0.11.0", "ac7a0cc03035317b6fef9fe94c97d7d9bd183a3e7ce1606aa0c175cfa8d1ba6d", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:ex_aws_s3, "~> 2.0", [hex: :ex_aws_s3, repo: "hexpm", optional: true]}, {:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:poison, "~> 2.2 or ~> 3.1", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "e91a8bd676fca716f6e46275ae81fb96c0bbc7a9d5b96cac511ae190588eddd0"}, - "arc_ecto": {:hex, :arc_ecto, "0.11.3", "52f278330fe3a29472ce5d9682514ca09eaed4b33453cbaedb5241a491464f7d", [:mix], [{:arc, "~> 0.11.0", [hex: :arc, repo: "hexpm", optional: false]}, {:ecto, ">= 2.1.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "24beed35003707434a778caece7d71e46e911d46d1e82e7787345264fc8e96d0"}, "artificery": {:hex, :artificery, "0.4.3", "0bc4260f988dcb9dda4b23f9fc3c6c8b99a6220a331534fdf5bf2fd0d4333b02", [:mix], [], "hexpm", "12e95333a30e20884e937abdbefa3e7f5e05609c2ba8cf37b33f000b9ffc0504"}, "bamboo": {:hex, :bamboo, "2.5.0", "973f5cb1471a1d2d7d9da7b8e4f6096afb6a133f85394631184fd40be8adb8ab", [:mix], [{:hackney, ">= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.4 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "35c8635ff6677a81ab7258944ff15739280f3254a041b6f0229dddeb9b90ad3d"}, "bamboo_phoenix": {:hex, :bamboo_phoenix, "1.0.0", "f3cc591ffb163ed0bf935d256f1f4645cd870cf436545601215745fb9cc9953f", [:mix], [{:bamboo, ">= 2.0.0", [hex: :bamboo, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.3.0", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "6db88fbb26019c84a47994bb2bd879c0887c29ce6c559bc6385fd54eb8b37dee"}, @@ -101,6 +99,8 @@ "timex": {:hex, :timex, "3.7.13", "0688ce11950f5b65e154e42b47bf67b15d3bc0e0c3def62199991b8a8079a1e2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.26", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "09588e0522669328e973b8b4fd8741246321b3f0d32735b589f78b136e6d4c54"}, "tzdata": {:hex, :tzdata, "1.1.3", "b1cef7bb6de1de90d4ddc25d33892b32830f907e7fc2fccd1e7e22778ab7dfbc", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "d4ca85575a064d29d4e94253ee95912edfb165938743dbf002acdf0dcecb0c28"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"}, + "waffle": {:hex, :waffle, "1.1.10", "0f847ed6f95349af258a90f0f70ffea02b3d3729c4eb78f6fae7bf776e91779e", [:mix], [{:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:ex_aws_s3, "~> 2.1", [hex: :ex_aws_s3, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "859ba6377b78f0a51bc9596227b194f26241efbbd408bd217450c22b0f359cc4"}, + "waffle_ecto": {:hex, :waffle_ecto, "0.0.12", "e5c17c49b071b903df71861c642093281123142dc4e9908c930db3e06795b040", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:waffle, "~> 1.0", [hex: :waffle, repo: "hexpm", optional: false]}], "hexpm", "585fe6371057066d2e8e3383ddd7a2437ff0668caf3f4cbf5a041e0de9837168"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.9", "43dc3ba6d89ef5dec5b1d0a39698436a1e856d000d84bf31a3149862b01a287f", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "5534d5c9adad3c18a0f58a9371220d75a803bf0b9a3d87e6fe072faaeed76a08"}, "xml_builder": {:hex, :xml_builder, "2.4.0", "b20d23077266c81f593360dc037ea398461dddb6638a329743da6c73afa56725", [:mix], [], "hexpm", "833e325bb997f032b5a1b740d2fd6feed3c18ca74627f9f5f30513a9ae1a232d"}, diff --git a/test/support/data_case.ex b/test/support/data_case.ex index 5b0a5c76f..74f0e116f 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -51,7 +51,7 @@ defmodule Cadet.DataCase do end @doc """ - A helper that builds a Plug.Upload struct to test Arc.Ecto fields + A helper that builds a Plug.Upload struct to test Waffle.Ecto fields """ def build_upload(path, content_type \\ "image\png") do %Plug.Upload{path: path, filename: Path.basename(path), content_type: content_type} From 23cefca90bb3fbbda6992f355bca6277d13825ed Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 22:48:53 +0800 Subject: [PATCH 23/68] Fix: isLlmGraded to ignore empty assessment prompts --- lib/cadet_web/admin_views/admin_assessments_view.ex | 2 +- lib/cadet_web/views/assessments_view.ex | 2 +- .../admin_controllers/admin_assessments_controller_test.exs | 4 ++-- test/cadet_web/controllers/assessments_controller_test.exs | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/cadet_web/admin_views/admin_assessments_view.ex b/lib/cadet_web/admin_views/admin_assessments_view.ex index f060722b0..f1d81d15a 100644 --- a/lib/cadet_web/admin_views/admin_assessments_view.ex +++ b/lib/cadet_web/admin_views/admin_assessments_view.ex @@ -33,7 +33,7 @@ defmodule CadetWeb.AdminAssessmentsView do hasVotingFeatures: :has_voting_features, hasTokenCounter: :has_token_counter, isVotingPublished: :is_voting_published, - isLlmGraded: &(&1.has_llm_questions || &1.llm_assessment_prompt != nil) + isLlmGraded: &(&1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""]) }) end diff --git a/lib/cadet_web/views/assessments_view.ex b/lib/cadet_web/views/assessments_view.ex index 343494949..5c22b1f72 100644 --- a/lib/cadet_web/views/assessments_view.ex +++ b/lib/cadet_web/views/assessments_view.ex @@ -36,7 +36,7 @@ defmodule CadetWeb.AssessmentsView do hasTokenCounter: :has_token_counter, isVotingPublished: :is_voting_published, hoursBeforeEarlyXpDecay: & &1.config.hours_before_early_xp_decay, - isLlmGraded: &(&1.has_llm_questions || &1.llm_assessment_prompt != nil) + isLlmGraded: &(&1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""]) }) end diff --git a/test/cadet_web/admin_controllers/admin_assessments_controller_test.exs b/test/cadet_web/admin_controllers/admin_assessments_controller_test.exs index ee76ac374..5368a3e8c 100644 --- a/test/cadet_web/admin_controllers/admin_assessments_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_assessments_controller_test.exs @@ -95,7 +95,7 @@ defmodule CadetWeb.AdminAssessmentsControllerTest do "hasVotingFeatures" => &1.has_voting_features, "hasTokenCounter" => &1.has_token_counter, "isVotingPublished" => false, - "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt != nil, + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""], } ) @@ -147,7 +147,7 @@ defmodule CadetWeb.AdminAssessmentsControllerTest do "hasVotingFeatures" => &1.has_voting_features, "hasTokenCounter" => &1.has_token_counter, "isVotingPublished" => false, - "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt != nil, + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""], } ) diff --git a/test/cadet_web/controllers/assessments_controller_test.exs b/test/cadet_web/controllers/assessments_controller_test.exs index cb4698cfe..efea07828 100644 --- a/test/cadet_web/controllers/assessments_controller_test.exs +++ b/test/cadet_web/controllers/assessments_controller_test.exs @@ -85,7 +85,7 @@ defmodule CadetWeb.AssessmentsControllerTest do "hasTokenCounter" => &1.has_token_counter, "isVotingPublished" => false, "hoursBeforeEarlyXpDecay" => &1.config.hours_before_early_xp_decay, - "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt != nil + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""] } ) @@ -177,7 +177,7 @@ defmodule CadetWeb.AssessmentsControllerTest do "hasTokenCounter" => &1.has_token_counter, "isVotingPublished" => false, "hoursBeforeEarlyXpDecay" => &1.config.hours_before_early_xp_decay, - "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt != nil, + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""], } ) @@ -300,7 +300,7 @@ defmodule CadetWeb.AssessmentsControllerTest do else &1.is_published end, - "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt != nil + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""] } ) From 5d899956ae09d5a0908184492c95366e77870b5f Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 22:57:24 +0800 Subject: [PATCH 24/68] test(assessments): add has_llm_questions aggregation coverage --- test/cadet/assessments/query_test.exs | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/cadet/assessments/query_test.exs b/test/cadet/assessments/query_test.exs index 5f01b6e0e..02e1c907c 100644 --- a/test/cadet/assessments/query_test.exs +++ b/test/cadet/assessments/query_test.exs @@ -28,4 +28,48 @@ defmodule Cadet.Assessments.QueryTest do assert result.max_xp == 1000 end + + test "all_assessments_with_aggregates sets has_llm_questions to true when any question has non-empty llm_prompt" do + course = insert(:course) + assessment = insert(:assessment, course: course) + + insert(:question, + assessment: assessment, + question: build(:programming_question_content, llm_prompt: "Provide AI feedback") + ) + + insert(:question, + assessment: assessment, + question: build(:programming_question_content, llm_prompt: nil) + ) + + result = + Query.all_assessments_with_aggregates(course.id) + |> where(id: ^assessment.id) + |> Repo.one() + + assert result.has_llm_questions == true + end + + test "all_assessments_with_aggregates sets has_llm_questions to false when all llm_prompt values are nil or empty" do + course = insert(:course) + assessment = insert(:assessment, course: course) + + insert(:question, + assessment: assessment, + question: build(:programming_question_content, llm_prompt: nil) + ) + + insert(:question, + assessment: assessment, + question: build(:programming_question_content, llm_prompt: "") + ) + + result = + Query.all_assessments_with_aggregates(course.id) + |> where(id: ^assessment.id) + |> Repo.one() + + assert result.has_llm_questions == false + end end From 85a26820d9911e549915ad4261872b435ad27f2e Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 23:28:54 +0800 Subject: [PATCH 25/68] test(llm-stats): add coverage for usage logging, stats aggregation, and feedback filtering --- test/cadet/llm_stats_test.exs | 242 ++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 test/cadet/llm_stats_test.exs diff --git a/test/cadet/llm_stats_test.exs b/test/cadet/llm_stats_test.exs new file mode 100644 index 000000000..c5a381533 --- /dev/null +++ b/test/cadet/llm_stats_test.exs @@ -0,0 +1,242 @@ +defmodule Cadet.LLMStatsTest do + use Cadet.DataCase + + alias Cadet.LLMStats + alias Cadet.LLMStats.LLMUsageLog + + describe "log_usage/1" do + test "inserts a usage log record" do + course = insert(:course) + assessment = insert(:assessment, course: course) + question = insert(:question, assessment: assessment, display_order: 1) + student = insert(:course_registration, course: course, role: :student) + submission = insert(:submission, assessment: assessment, student: student) + answer = insert(:answer, submission: submission, question: question) + user = insert(:user) + + attrs = %{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question.id, + answer_id: answer.id, + submission_id: submission.id, + user_id: user.id + } + + assert {:ok, usage_log} = LLMStats.log_usage(attrs) + assert usage_log.course_id == course.id + assert usage_log.assessment_id == assessment.id + assert usage_log.question_id == question.id + assert usage_log.answer_id == answer.id + assert usage_log.submission_id == submission.id + assert usage_log.user_id == user.id + + assert Repo.get(LLMUsageLog, usage_log.id) + end + end + + describe "get_assessment_statistics/2" do + test "returns aggregate and per-question statistics scoped to assessment" do + course = insert(:course) + assessment = insert(:assessment, course: course) + question_1 = insert(:question, assessment: assessment, display_order: 1) + question_2 = insert(:question, assessment: assessment, display_order: 2) + + student_1 = insert(:course_registration, course: course, role: :student) + student_2 = insert(:course_registration, course: course, role: :student) + submission_1 = insert(:submission, assessment: assessment, student: student_1) + submission_2 = insert(:submission, assessment: assessment, student: student_2) + + answer_11 = insert(:answer, submission: submission_1, question: question_1) + answer_12 = insert(:answer, submission: submission_1, question: question_2) + answer_21 = insert(:answer, submission: submission_2, question: question_1) + + user_1 = insert(:user) + user_2 = insert(:user) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + answer_id: answer_11.id, + submission_id: submission_1.id, + user_id: user_1.id + }) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + answer_id: answer_11.id, + submission_id: submission_1.id, + user_id: user_1.id + }) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + answer_id: answer_21.id, + submission_id: submission_2.id, + user_id: user_2.id + }) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_2.id, + answer_id: answer_12.id, + submission_id: submission_1.id, + user_id: user_1.id + }) + + other_course = insert(:course) + other_assessment = insert(:assessment, course: other_course) + other_question = insert(:question, assessment: other_assessment, display_order: 1) + other_student = insert(:course_registration, course: other_course, role: :student) + other_submission = insert(:submission, assessment: other_assessment, student: other_student) + other_answer = insert(:answer, submission: other_submission, question: other_question) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: other_course.id, + assessment_id: other_assessment.id, + question_id: other_question.id, + answer_id: other_answer.id, + submission_id: other_submission.id, + user_id: user_1.id + }) + + stats = LLMStats.get_assessment_statistics(course.id, assessment.id) + + assert stats.total_uses == 4 + assert stats.unique_submissions == 2 + assert stats.unique_users == 2 + + assert [q1_stats, q2_stats] = stats.questions + + assert q1_stats.question_id == question_1.id + assert q1_stats.display_order == 1 + assert q1_stats.total_uses == 3 + assert q1_stats.unique_submissions == 2 + assert q1_stats.unique_users == 2 + + assert q2_stats.question_id == question_2.id + assert q2_stats.display_order == 2 + assert q2_stats.total_uses == 1 + assert q2_stats.unique_submissions == 1 + assert q2_stats.unique_users == 1 + end + end + + describe "get_question_statistics/3" do + test "returns statistics scoped to one question" do + course = insert(:course) + assessment = insert(:assessment, course: course) + question_1 = insert(:question, assessment: assessment, display_order: 1) + question_2 = insert(:question, assessment: assessment, display_order: 2) + + student_1 = insert(:course_registration, course: course, role: :student) + student_2 = insert(:course_registration, course: course, role: :student) + submission_1 = insert(:submission, assessment: assessment, student: student_1) + submission_2 = insert(:submission, assessment: assessment, student: student_2) + + answer_11 = insert(:answer, submission: submission_1, question: question_1) + answer_21 = insert(:answer, submission: submission_2, question: question_1) + answer_12 = insert(:answer, submission: submission_1, question: question_2) + + user_1 = insert(:user) + user_2 = insert(:user) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + answer_id: answer_11.id, + submission_id: submission_1.id, + user_id: user_1.id + }) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + answer_id: answer_11.id, + submission_id: submission_1.id, + user_id: user_1.id + }) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + answer_id: answer_21.id, + submission_id: submission_2.id, + user_id: user_2.id + }) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_2.id, + answer_id: answer_12.id, + submission_id: submission_1.id, + user_id: user_1.id + }) + + stats = LLMStats.get_question_statistics(course.id, assessment.id, question_1.id) + + assert stats.total_uses == 3 + assert stats.unique_submissions == 2 + assert stats.unique_users == 2 + end + end + + describe "get_feedback/3" do + test "filters by question_id when provided" do + course = insert(:course) + assessment = insert(:assessment, course: course) + question_1 = insert(:question, assessment: assessment, display_order: 1) + question_2 = insert(:question, assessment: assessment, display_order: 2) + user_1 = insert(:user, name: "Alice") + user_2 = insert(:user, name: "Bob") + + assert {:ok, _} = + LLMStats.submit_feedback(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + user_id: user_1.id, + rating: 5, + body: "Very helpful" + }) + + assert {:ok, _} = + LLMStats.submit_feedback(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_2.id, + user_id: user_2.id, + rating: 3, + body: "Could be clearer" + }) + + unfiltered = LLMStats.get_feedback(course.id, assessment.id) + filtered = LLMStats.get_feedback(course.id, assessment.id, question_1.id) + + assert Enum.count(unfiltered) == 2 + assert Enum.count(filtered) == 1 + + assert [%{question_id: qid, user_name: "Alice", rating: 5, body: "Very helpful"}] = filtered + assert qid == question_1.id + end + end +end From a4eb93190a96e13a7a79d135453bf47d5d29652d Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 23:37:21 +0800 Subject: [PATCH 26/68] test(admin): add request tests for LLM stats and feedback endpoints --- .../admin_llm_stats_controller_test.exs | 317 ++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs diff --git a/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs b/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs new file mode 100644 index 000000000..1295f5505 --- /dev/null +++ b/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs @@ -0,0 +1,317 @@ +defmodule CadetWeb.AdminLLMStatsControllerTest do + use CadetWeb.ConnCase + + alias Cadet.LLMStats + alias Cadet.Repo + alias Cadet.Courses.Course + + describe "GET /v2/courses/:course_id/admin/llm-stats/:assessment_id" do + test "401 when not logged in", %{conn: conn} do + course = insert(:course) + assessment = insert(:assessment, course: course) + + conn = get(conn, assessment_stats_url(course.id, assessment.id)) + assert response(conn, 401) =~ "Unauthorised" + end + + @tag authenticate: :student + test "403 for students", %{conn: conn} do + course_id = conn.assigns.course_id + course = Repo.get!(Course, course_id) + assessment = insert(:assessment, course: course) + + conn = get(conn, assessment_stats_url(course_id, assessment.id)) + assert response(conn, 403) =~ "Forbidden" + end + + @tag authenticate: :staff + test "returns assessment statistics with per-question breakdown", %{conn: conn} do + %{assessment: assessment, question_1: question_1, question_2: question_2} = seed_usage_logs(conn) + + resp = + conn + |> get(assessment_stats_url(conn.assigns.course_id, assessment.id)) + |> json_response(200) + + assert resp["total_uses"] == 4 + assert resp["unique_submissions"] == 2 + assert resp["unique_users"] == 2 + + assert [q1_stats, q2_stats] = resp["questions"] + + assert q1_stats["question_id"] == question_1.id + assert q1_stats["display_order"] == question_1.display_order + assert q1_stats["total_uses"] == 3 + assert q1_stats["unique_submissions"] == 2 + assert q1_stats["unique_users"] == 2 + + assert q2_stats["question_id"] == question_2.id + assert q2_stats["display_order"] == question_2.display_order + assert q2_stats["total_uses"] == 1 + assert q2_stats["unique_submissions"] == 1 + assert q2_stats["unique_users"] == 1 + end + end + + describe "GET /v2/courses/:course_id/admin/llm-stats/:assessment_id/:question_id" do + test "401 when not logged in", %{conn: conn} do + course = insert(:course) + assessment = insert(:assessment, course: course) + question = insert(:question, assessment: assessment, display_order: 1) + + conn = get(conn, question_stats_url(course.id, assessment.id, question.id)) + assert response(conn, 401) =~ "Unauthorised" + end + + @tag authenticate: :student + test "403 for students", %{conn: conn} do + course_id = conn.assigns.course_id + course = Repo.get!(Course, course_id) + assessment = insert(:assessment, course: course) + question = insert(:question, assessment: assessment, display_order: 1) + + conn = get(conn, question_stats_url(course_id, assessment.id, question.id)) + assert response(conn, 403) =~ "Forbidden" + end + + @tag authenticate: :staff + test "returns question-level statistics", %{conn: conn} do + %{assessment: assessment, question_1: question_1} = seed_usage_logs(conn) + + resp = + conn + |> get(question_stats_url(conn.assigns.course_id, assessment.id, question_1.id)) + |> json_response(200) + + assert resp == %{ + "total_uses" => 3, + "unique_submissions" => 2, + "unique_users" => 2 + } + end + end + + describe "GET /v2/courses/:course_id/admin/llm-stats/:assessment_id/feedback" do + test "401 when not logged in", %{conn: conn} do + course = insert(:course) + assessment = insert(:assessment, course: course) + + conn = get(conn, feedback_url(course.id, assessment.id)) + assert response(conn, 401) =~ "Unauthorised" + end + + @tag authenticate: :student + test "403 for students", %{conn: conn} do + course_id = conn.assigns.course_id + course = Repo.get!(Course, course_id) + assessment = insert(:assessment, course: course) + + conn = get(conn, feedback_url(course_id, assessment.id)) + assert response(conn, 403) =~ "Forbidden" + end + + @tag authenticate: :staff + test "returns all feedback when question_id is absent", %{conn: conn} do + %{assessment: assessment} = seed_feedback(conn) + + resp = + conn + |> get(feedback_url(conn.assigns.course_id, assessment.id)) + |> json_response(200) + + assert length(resp) == 2 + assert Enum.all?(resp, &Map.has_key?(&1, "id")) + assert Enum.all?(resp, &Map.has_key?(&1, "rating")) + assert Enum.all?(resp, &Map.has_key?(&1, "body")) + assert Enum.all?(resp, &Map.has_key?(&1, "user_name")) + assert Enum.all?(resp, &Map.has_key?(&1, "question_id")) + assert Enum.all?(resp, &Map.has_key?(&1, "inserted_at")) + end + + @tag authenticate: :staff + test "filters feedback by question_id query param", %{conn: conn} do + %{assessment: assessment, question_1: question_1} = seed_feedback(conn) + + resp = + conn + |> get(feedback_url(conn.assigns.course_id, assessment.id), %{"question_id" => question_1.id}) + |> json_response(200) + + assert length(resp) == 1 + + [entry] = resp + assert entry["question_id"] == question_1.id + assert entry["user_name"] == "Alice" + assert entry["rating"] == 5 + assert entry["body"] == "Very helpful" + end + end + + describe "POST /v2/courses/:course_id/admin/llm-stats/:assessment_id/feedback" do + test "401 when not logged in", %{conn: conn} do + course = insert(:course) + assessment = insert(:assessment, course: course) + + conn = post(conn, feedback_url(course.id, assessment.id), %{"rating" => 5, "body" => "Great"}) + assert response(conn, 401) =~ "Unauthorised" + end + + @tag authenticate: :student + test "403 for students", %{conn: conn} do + course_id = conn.assigns.course_id + course = Repo.get!(Course, course_id) + assessment = insert(:assessment, course: course) + + conn = post(conn, feedback_url(course_id, assessment.id), %{"rating" => 5, "body" => "Great"}) + assert response(conn, 403) =~ "Forbidden" + end + + @tag authenticate: :staff + test "creates feedback successfully", %{conn: conn} do + course_id = conn.assigns.course_id + course = Repo.get!(Course, course_id) + assessment = insert(:assessment, course: course) + question = insert(:question, assessment: assessment, display_order: 1) + + resp = + conn + |> post(feedback_url(course_id, assessment.id), %{ + "question_id" => question.id, + "rating" => 4, + "body" => "Reasonably useful" + }) + |> json_response(201) + + assert resp == %{"message" => "Feedback submitted successfully"} + + [saved_feedback] = LLMStats.get_feedback(course_id, assessment.id, question.id) + assert saved_feedback.rating == 4 + assert saved_feedback.body == "Reasonably useful" + assert saved_feedback.user_name == conn.assigns.current_user.name + end + + @tag authenticate: :staff + test "returns 400 when payload is invalid", %{conn: conn} do + course_id = conn.assigns.course_id + course = Repo.get!(Course, course_id) + assessment = insert(:assessment, course: course) + + resp = + conn + |> post(feedback_url(course_id, assessment.id), %{"rating" => 6}) + |> json_response(400) + + assert resp == %{"error" => "Failed to submit feedback"} + end + end + + defp seed_usage_logs(conn) do + course = Repo.get!(Course, conn.assigns.course_id) + assessment = insert(:assessment, course: course) + question_1 = insert(:question, assessment: assessment, display_order: 1) + question_2 = insert(:question, assessment: assessment, display_order: 2) + + student_1 = insert(:course_registration, course: course, role: :student) + student_2 = insert(:course_registration, course: course, role: :student) + + submission_1 = insert(:submission, assessment: assessment, student: student_1) + submission_2 = insert(:submission, assessment: assessment, student: student_2) + + answer_11 = insert(:answer, submission: submission_1, question: question_1) + answer_12 = insert(:answer, submission: submission_1, question: question_2) + answer_21 = insert(:answer, submission: submission_2, question: question_1) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + answer_id: answer_11.id, + submission_id: submission_1.id, + user_id: student_1.user_id + }) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + answer_id: answer_11.id, + submission_id: submission_1.id, + user_id: student_1.user_id + }) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + answer_id: answer_21.id, + submission_id: submission_2.id, + user_id: student_2.user_id + }) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_2.id, + answer_id: answer_12.id, + submission_id: submission_1.id, + user_id: student_1.user_id + }) + + %{ + assessment: assessment, + question_1: question_1, + question_2: question_2 + } + end + + defp seed_feedback(conn) do + course = Repo.get!(Course, conn.assigns.course_id) + assessment = insert(:assessment, course: course) + question_1 = insert(:question, assessment: assessment, display_order: 1) + question_2 = insert(:question, assessment: assessment, display_order: 2) + user_1 = insert(:user, name: "Alice") + user_2 = insert(:user, name: "Bob") + + assert {:ok, _} = + LLMStats.submit_feedback(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + user_id: user_1.id, + rating: 5, + body: "Very helpful" + }) + + assert {:ok, _} = + LLMStats.submit_feedback(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_2.id, + user_id: user_2.id, + rating: 3, + body: "Could be clearer" + }) + + %{ + assessment: assessment, + question_1: question_1, + question_2: question_2 + } + end + + defp assessment_stats_url(course_id, assessment_id) do + "/v2/courses/#{course_id}/admin/llm-stats/#{assessment_id}" + end + + defp question_stats_url(course_id, assessment_id, question_id) do + "/v2/courses/#{course_id}/admin/llm-stats/#{assessment_id}/#{question_id}" + end + + defp feedback_url(course_id, assessment_id) do + "/v2/courses/#{course_id}/admin/llm-stats/#{assessment_id}/feedback" + end +end From eddad9206a5c9b35a5640b741256751a6175fc4b Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 23:46:01 +0800 Subject: [PATCH 27/68] Fix: Handle LLM usage log insert failures in AI comments controller --- .../controllers/generate_ai_comments.ex | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 7c8ac9254..b48cc957f 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -235,15 +235,25 @@ defmodule CadetWeb.AICodeAnalysisController do content ) - # Log LLM usage for statistics - LLMStats.log_usage(%{ + usage_attrs = %{ course_id: course_id, assessment_id: answer.question.assessment_id, question_id: answer.question_id, answer_id: answer.id, submission_id: answer.submission_id, user_id: conn.assigns.course_reg.user_id - }) + } + + # Log LLM usage for statistics (non-blocking for response generation) + case LLMStats.log_usage(usage_attrs) do + {:ok, _usage_log} -> + :ok + + {:error, changeset} -> + Logger.error( + "Failed to log LLM usage to database: #{inspect(changeset.errors)} attrs=#{inspect(usage_attrs)}" + ) + end comments_list = String.split(content, "|||") From 7dfed65874f2c06f776920aa44751daba26e5b92 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Tue, 17 Mar 2026 23:52:01 +0800 Subject: [PATCH 28/68] Fix: formatting issues --- lib/cadet/assessments/query.ex | 6 +++++- .../admin_llm_stats_controller.ex | 5 ++++- lib/cadet_web/admin_views/admin_grading_view.ex | 4 +++- lib/cadet_web/controllers/generate_ai_comments.ex | 7 +++---- test/cadet/ai_comments_test.exs | 5 ++++- .../admin_assessments_controller_test.exs | 4 ++-- .../admin_llm_stats_controller_test.exs | 15 +++++++++++---- .../controllers/assessments_controller_test.exs | 2 +- 8 files changed, 33 insertions(+), 15 deletions(-) diff --git a/lib/cadet/assessments/query.ex b/lib/cadet/assessments/query.ex index a75612d45..b93c20d51 100644 --- a/lib/cadet/assessments/query.ex +++ b/lib/cadet/assessments/query.ex @@ -52,7 +52,11 @@ defmodule Cadet.Assessments.Query do max_xp: sum(q.max_xp), question_count: count(q.id), has_llm_questions: - fragment("bool_or(? ->> 'llm_prompt' IS NOT NULL AND ? ->> 'llm_prompt' != '')", q.question, q.question) + fragment( + "bool_or(? ->> 'llm_prompt' IS NOT NULL AND ? ->> 'llm_prompt' != '')", + q.question, + q.question + ) }) end end diff --git a/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex b/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex index cd4ea140c..5ea0a0fac 100644 --- a/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex @@ -44,7 +44,10 @@ defmodule CadetWeb.AdminLLMStatsController do POST /admin/llm-stats/:assessment_id/feedback Submits new feedback for the LLM feature on an assessment (optionally for a specific question). """ - def submit_feedback(conn, %{"course_id" => course_id, "assessment_id" => assessment_id} = params) do + def submit_feedback( + conn, + %{"course_id" => course_id, "assessment_id" => assessment_id} = params + ) do user = conn.assigns[:current_user] attrs = %{ diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index 69a46f343..fd9156f24 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -182,7 +182,9 @@ defmodule CadetWeb.AdminGradingView do end case latest_comment do - nil -> nil + nil -> + nil + comment -> selected_indices = comment.selected_indices || [] diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index b48cc957f..2fdd0d393 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -308,7 +308,8 @@ defmodule CadetWeb.AICodeAnalysisController do edits = Map.get(params, "edits", %{}) with {:ok, answer_id_parsed} <- parse_answer_id(answer_id), - ai_comment when not is_nil(ai_comment) <- AIComments.get_latest_ai_comment(answer_id_parsed), + ai_comment when not is_nil(ai_comment) <- + AIComments.get_latest_ai_comment(answer_id_parsed), {:ok, _updated} <- AIComments.save_selected_comments(answer_id_parsed, selected_indices, editor_id), {:ok, parsed_edits} <- parse_edits(edits) do @@ -449,9 +450,7 @@ defmodule CadetWeb.AICodeAnalysisController do SaveChosenCommentsBody: swagger_schema do properties do - selected_indices(Schema.array(:integer), "Indices of chosen comments", - required: true - ) + selected_indices(Schema.array(:integer), "Indices of chosen comments", required: true) edits(:object, "Map of comment index to edited text") end diff --git a/test/cadet/ai_comments_test.exs b/test/cadet/ai_comments_test.exs index 8c9120fb0..d29bcc5a0 100644 --- a/test/cadet/ai_comments_test.exs +++ b/test/cadet/ai_comments_test.exs @@ -23,7 +23,10 @@ defmodule Cadet.AICommentsTest do {:ok, ai_comment: ai_comment, editor: editor} end - test "creates distinct version numbers for concurrent edits", %{ai_comment: ai_comment, editor: editor} do + test "creates distinct version numbers for concurrent edits", %{ + ai_comment: ai_comment, + editor: editor + } do parent = self() create_version = fn content -> diff --git a/test/cadet_web/admin_controllers/admin_assessments_controller_test.exs b/test/cadet_web/admin_controllers/admin_assessments_controller_test.exs index 5368a3e8c..62b39ce58 100644 --- a/test/cadet_web/admin_controllers/admin_assessments_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_assessments_controller_test.exs @@ -95,7 +95,7 @@ defmodule CadetWeb.AdminAssessmentsControllerTest do "hasVotingFeatures" => &1.has_voting_features, "hasTokenCounter" => &1.has_token_counter, "isVotingPublished" => false, - "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""], + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""] } ) @@ -147,7 +147,7 @@ defmodule CadetWeb.AdminAssessmentsControllerTest do "hasVotingFeatures" => &1.has_voting_features, "hasTokenCounter" => &1.has_token_counter, "isVotingPublished" => false, - "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""], + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""] } ) diff --git a/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs b/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs index 1295f5505..c499ed194 100644 --- a/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs @@ -26,7 +26,8 @@ defmodule CadetWeb.AdminLLMStatsControllerTest do @tag authenticate: :staff test "returns assessment statistics with per-question breakdown", %{conn: conn} do - %{assessment: assessment, question_1: question_1, question_2: question_2} = seed_usage_logs(conn) + %{assessment: assessment, question_1: question_1, question_2: question_2} = + seed_usage_logs(conn) resp = conn @@ -134,7 +135,9 @@ defmodule CadetWeb.AdminLLMStatsControllerTest do resp = conn - |> get(feedback_url(conn.assigns.course_id, assessment.id), %{"question_id" => question_1.id}) + |> get(feedback_url(conn.assigns.course_id, assessment.id), %{ + "question_id" => question_1.id + }) |> json_response(200) assert length(resp) == 1 @@ -152,7 +155,9 @@ defmodule CadetWeb.AdminLLMStatsControllerTest do course = insert(:course) assessment = insert(:assessment, course: course) - conn = post(conn, feedback_url(course.id, assessment.id), %{"rating" => 5, "body" => "Great"}) + conn = + post(conn, feedback_url(course.id, assessment.id), %{"rating" => 5, "body" => "Great"}) + assert response(conn, 401) =~ "Unauthorised" end @@ -162,7 +167,9 @@ defmodule CadetWeb.AdminLLMStatsControllerTest do course = Repo.get!(Course, course_id) assessment = insert(:assessment, course: course) - conn = post(conn, feedback_url(course_id, assessment.id), %{"rating" => 5, "body" => "Great"}) + conn = + post(conn, feedback_url(course_id, assessment.id), %{"rating" => 5, "body" => "Great"}) + assert response(conn, 403) =~ "Forbidden" end diff --git a/test/cadet_web/controllers/assessments_controller_test.exs b/test/cadet_web/controllers/assessments_controller_test.exs index efea07828..f037b0292 100644 --- a/test/cadet_web/controllers/assessments_controller_test.exs +++ b/test/cadet_web/controllers/assessments_controller_test.exs @@ -177,7 +177,7 @@ defmodule CadetWeb.AssessmentsControllerTest do "hasTokenCounter" => &1.has_token_counter, "isVotingPublished" => false, "hoursBeforeEarlyXpDecay" => &1.config.hours_before_early_xp_decay, - "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""], + "isLlmGraded" => &1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""] } ) From 7eddeb08e49ce4590fe3108a2b4067335b7da545 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Wed, 18 Mar 2026 00:15:38 +0800 Subject: [PATCH 29/68] Fix: more formatting issues --- lib/cadet/ai_comments.ex | 59 ++++++++++--------- .../admin_llm_stats_controller.ex | 4 +- test/cadet/assessments/query_test.exs | 6 +- .../admin_llm_stats_controller_test.exs | 4 +- 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 9b444ff11..7ca71234b 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -81,36 +81,39 @@ defmodule Cadet.AIComments do Automatically determines the next version number. """ def create_comment_version(ai_comment_id, comment_index, content, editor_id) do - Repo.transaction(fn -> - # Serialize version creation per (ai_comment_id, comment_index) to avoid duplicate version numbers. - case Repo.query("SELECT pg_advisory_xact_lock($1, $2)", [ai_comment_id, comment_index]) do - {:ok, _} -> - next_version = - Repo.one( - from(v in AICommentVersion, - where: v.ai_comment_id == ^ai_comment_id and v.comment_index == ^comment_index, - select: coalesce(max(v.version_number), 0) - ) - ) + 1 + transaction_result = + Repo.transaction(fn -> + # Serialize version creation per (ai_comment_id, comment_index) + # to avoid duplicate version numbers. + case Repo.query("SELECT pg_advisory_xact_lock($1, $2)", [ai_comment_id, comment_index]) do + {:ok, _} -> + next_version = + Repo.one( + from(v in AICommentVersion, + where: v.ai_comment_id == ^ai_comment_id and v.comment_index == ^comment_index, + select: coalesce(max(v.version_number), 0) + ) + ) + 1 - case %AICommentVersion{} - |> AICommentVersion.changeset(%{ - ai_comment_id: ai_comment_id, - comment_index: comment_index, - version_number: next_version, - content: content, - editor_id: editor_id - }) - |> Repo.insert() do - {:ok, version} -> version - {:error, changeset} -> Repo.rollback(changeset) - end + case %AICommentVersion{} + |> AICommentVersion.changeset(%{ + ai_comment_id: ai_comment_id, + comment_index: comment_index, + version_number: next_version, + content: content, + editor_id: editor_id + }) + |> Repo.insert() do + {:ok, version} -> version + {:error, changeset} -> Repo.rollback(changeset) + end - {:error, error} -> - Repo.rollback(error) - end - end) - |> case do + {:error, error} -> + Repo.rollback(error) + end + end) + + case transaction_result do {:ok, version} -> {:ok, version} {:error, reason} -> {:error, reason} end diff --git a/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex b/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex index 5ea0a0fac..3046d6c48 100644 --- a/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex @@ -34,7 +34,7 @@ defmodule CadetWeb.AdminLLMStatsController do GET /admin/llm-stats/:assessment_id/feedback Returns feedback for an assessment, optionally filtered by question_id query param. """ - def get_feedback(conn, %{"course_id" => course_id, "assessment_id" => assessment_id} = params) do + def get_feedback(conn, params = %{"course_id" => course_id, "assessment_id" => assessment_id}) do question_id = Map.get(params, "question_id") feedback = LLMStats.get_feedback(course_id, assessment_id, question_id) json(conn, feedback) @@ -46,7 +46,7 @@ defmodule CadetWeb.AdminLLMStatsController do """ def submit_feedback( conn, - %{"course_id" => course_id, "assessment_id" => assessment_id} = params + params = %{"course_id" => course_id, "assessment_id" => assessment_id} ) do user = conn.assigns[:current_user] diff --git a/test/cadet/assessments/query_test.exs b/test/cadet/assessments/query_test.exs index 02e1c907c..aed849a09 100644 --- a/test/cadet/assessments/query_test.exs +++ b/test/cadet/assessments/query_test.exs @@ -44,7 +44,8 @@ defmodule Cadet.Assessments.QueryTest do ) result = - Query.all_assessments_with_aggregates(course.id) + course.id + |> Query.all_assessments_with_aggregates() |> where(id: ^assessment.id) |> Repo.one() @@ -66,7 +67,8 @@ defmodule Cadet.Assessments.QueryTest do ) result = - Query.all_assessments_with_aggregates(course.id) + course.id + |> Query.all_assessments_with_aggregates() |> where(id: ^assessment.id) |> Repo.one() diff --git a/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs b/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs index c499ed194..2095f4c5a 100644 --- a/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs @@ -1,9 +1,7 @@ defmodule CadetWeb.AdminLLMStatsControllerTest do use CadetWeb.ConnCase - alias Cadet.LLMStats - alias Cadet.Repo - alias Cadet.Courses.Course + alias Cadet.{LLMStats, Repo, Courses.Course} describe "GET /v2/courses/:course_id/admin/llm-stats/:assessment_id" do test "401 when not logged in", %{conn: conn} do From b2b6fce9e70fa1d5408620a715040847c8b0efe0 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Wed, 18 Mar 2026 00:39:53 +0800 Subject: [PATCH 30/68] fix: remove unreachable parse_answer_id fallback to satisfy Dialyzer --- lib/cadet/ai_comments.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 7ca71234b..213c97ee7 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -143,4 +143,4 @@ defmodule Cadet.AIComments do ) ) end -end +end \ No newline at end of file From 56b9cb6981f0aeea0932b88664ed41c85203f5ab Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Wed, 18 Mar 2026 00:43:30 +0800 Subject: [PATCH 31/68] Fix: save_chosen_comments order to validate edits before persisting selection --- lib/cadet_web/controllers/generate_ai_comments.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 2fdd0d393..ae123fcdd 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -310,9 +310,9 @@ defmodule CadetWeb.AICodeAnalysisController do with {:ok, answer_id_parsed} <- parse_answer_id(answer_id), ai_comment when not is_nil(ai_comment) <- AIComments.get_latest_ai_comment(answer_id_parsed), + {:ok, parsed_edits} <- parse_edits(edits), {:ok, _updated} <- - AIComments.save_selected_comments(answer_id_parsed, selected_indices, editor_id), - {:ok, parsed_edits} <- parse_edits(edits) do + AIComments.save_selected_comments(answer_id_parsed, selected_indices, editor_id) do # Create version entries for each edit version_results = Enum.map(parsed_edits, fn {index, edited_text} -> @@ -365,8 +365,6 @@ defmodule CadetWeb.AICodeAnalysisController do end end - defp parse_answer_id(_), do: {:error, :invalid_answer_id} - defp parse_edits(edits) when is_map(edits) do edits |> Enum.reduce_while({:ok, []}, fn {index_str, edited_text}, {:ok, acc} -> From 657383d12dd35e4c27009c6db730e710abf723f7 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Wed, 18 Mar 2026 00:46:03 +0800 Subject: [PATCH 32/68] Fix: formatting issue --- lib/cadet/ai_comments.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 213c97ee7..7ca71234b 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -143,4 +143,4 @@ defmodule Cadet.AIComments do ) ) end -end \ No newline at end of file +end From 4556433be33562a3f69e2f2816247d85af2e62d0 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Fri, 20 Mar 2026 20:51:58 +0800 Subject: [PATCH 33/68] feat: implement AI token cost tracking and stats UI --- lib/cadet/assessments/assessment.ex | 31 ++++++++++- lib/cadet/assessments/assessments.ex | 55 +++++++++++++++++++ lib/cadet/llm_stats.ex | 26 ++++++++- .../controllers/generate_ai_comments.ex | 5 +- lib/cadet_web/views/assessments_view.ex | 8 ++- ..._add_detailed_llm_costs_to_assessments.exs | 35 ++++++++++++ ...4651_repair_llm_columns_on_assessments.exs | 7 +++ 7 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs create mode 100644 priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs diff --git a/lib/cadet/assessments/assessment.ex b/lib/cadet/assessments/assessment.ex index 0b1cbdac2..c13371127 100644 --- a/lib/cadet/assessments/assessment.ex +++ b/lib/cadet/assessments/assessment.ex @@ -38,6 +38,12 @@ defmodule Cadet.Assessments.Assessment do field(:has_token_counter, :boolean, default: false) field(:has_voting_features, :boolean, default: false) field(:llm_assessment_prompt, :string, default: nil) + field(:llm_input_cost, :decimal) + field(:llm_output_cost, :decimal) + field(:llm_total_input_tokens, :integer, default: 0) + field(:llm_total_output_tokens, :integer, default: 0) + field(:llm_total_cached_tokens, :integer, default: 0) + field(:llm_total_cost, :decimal, default: Decimal.new("0.0")) belongs_to(:config, AssessmentConfig) belongs_to(:course, Course) @@ -48,7 +54,10 @@ defmodule Cadet.Assessments.Assessment do @required_fields ~w(title open_at close_at number course_id config_id max_team_size)a @optional_fields ~w(reading summary_short summary_long - is_published story cover_picture access password has_token_counter has_voting_features llm_assessment_prompt)a + is_published story cover_picture access password has_token_counter + has_voting_features llm_assessment_prompt + llm_input_cost llm_output_cost llm_total_input_tokens + llm_total_output_tokens llm_total_cached_tokens llm_total_cost)a @optional_file_fields ~w(mission_pdf)a def changeset(assessment, params) do @@ -61,12 +70,14 @@ defmodule Cadet.Assessments.Assessment do |> cast_attachments(params, @optional_file_fields) |> cast(params, @required_fields ++ @optional_fields) |> validate_required(@required_fields) + # ADD THIS LINE HERE to apply your defaults + |> put_default_costs() |> add_belongs_to_id_from_model([:config, :course], params) |> foreign_key_constraint(:config_id) |> foreign_key_constraint(:course_id) |> unique_constraint([:number, :course_id]) - |> validate_config_course - |> validate_open_close_date + |> validate_config_course() + |> validate_open_close_date() |> validate_number(:max_team_size, greater_than_or_equal_to: 1) end @@ -87,6 +98,20 @@ defmodule Cadet.Assessments.Assessment do end end + defp put_default_costs(changeset) do + changeset + |> put_fallback(:llm_input_cost, Decimal.new("3.20")) + |> put_fallback(:llm_output_cost, Decimal.new("12.80")) + end + + defp put_fallback(changeset, field, default_val) do + if get_field(changeset, field) == nil do + put_change(changeset, field, default_val) + else + changeset + end + end + defp validate_open_close_date(changeset) do validate_change(changeset, :open_at, fn :open_at, open_at -> if Timex.before?(open_at, get_field(changeset, :close_at)) do diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 426fc8914..d7dd608ac 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -1451,6 +1451,14 @@ defmodule Cadet.Assessments do end end + @doc """ + Deletes all AI comment logs associated with the given list of answer IDs. + """ + def delete_comments_for_answers(answer_ids) when is_list(answer_ids) do + query = from(c in "ai_comment_logs", where: c.answer_id in ^answer_ids) + Repo.delete_all(query) + end + @dialyzer {:nowarn_function, unsubmit_submission: 2} def unsubmit_submission( submission_id, @@ -1529,6 +1537,18 @@ defmodule Cadet.Assessments do end end) end) + |> Multi.run(:delete_ai_comments, fn _repo, _ -> + Logger.info("Deleting AI comments for submission #{submission_id}") + + answer_ids = + Answer + |> where(submission_id: ^submission_id) + |> select([a], a.id) + |> Repo.all() + + delete_comments_for_answers(answer_ids) + {:ok, nil} + end) |> Repo.transaction() case submission.student_id do @@ -3610,4 +3630,39 @@ defmodule Cadet.Assessments do Repo.one(query) end + + def update_llm_usage_and_cost(assessment_id, usage) do + assessment = Repo.get(Assessment, assessment_id) + + # Try string keys first, fall back to atom keys + prompt = usage["prompt_tokens"] || usage[:prompt_tokens] || 0 + completion = usage["completion_tokens"] || usage[:completion_tokens] || 0 + + # Try to find cached tokens in the nested map + details = usage["prompt_tokens_details"] || usage[:prompt_tokens_details] || %{} + cached = details["cached_tokens"] || details[:cached_tokens] || 0 + + # Logic: Use the keyed-in value if it's > 0, otherwise use SGD defaults + million = Decimal.new(1_000_000) + input_rate_1m = if assessment.llm_input_cost && Decimal.gt?(assessment.llm_input_cost, 0), + do: assessment.llm_input_cost, else: Decimal.new("3.20") + output_rate_1m = if assessment.llm_output_cost && Decimal.gt?(assessment.llm_output_cost, 0), + do: assessment.llm_output_cost, else: Decimal.new("12.80") + + # Math: (Tokens * Rate) / 1,000,000 + new_cost = Decimal.add( + Decimal.div(Decimal.mult(Decimal.new(prompt), input_rate_1m), million), + Decimal.div(Decimal.mult(Decimal.new(completion), output_rate_1m), million) + ) + + # UPDATE CALL + assessment + |> Assessment.changeset(%{ + llm_total_input_tokens: (assessment.llm_total_input_tokens || 0) + prompt, + llm_total_output_tokens: (assessment.llm_total_output_tokens || 0) + completion, + llm_total_cached_tokens: (assessment.llm_total_cached_tokens || 0) + cached, + llm_total_cost: Decimal.add(assessment.llm_total_cost || Decimal.new("0"), new_cost) + }) + |> Repo.update() +end end diff --git a/lib/cadet/llm_stats.ex b/lib/cadet/llm_stats.ex index 0987af480..b0b6eec31 100644 --- a/lib/cadet/llm_stats.ex +++ b/lib/cadet/llm_stats.ex @@ -33,6 +33,10 @@ defmodule Cadet.LLMStats do - unique_submissions: unique submissions that had LLM used - unique_users: unique users who used the feature - questions: per-question breakdown with stats + - llm_total_cost: Total cost in SGD + - llm_total_input_tokens: Total standard input tokens + - llm_total_output_tokens: Total output tokens + - llm_total_cached_tokens: Total cached input tokens """ def get_assessment_statistics(course_id, assessment_id) do base = @@ -74,11 +78,31 @@ defmodule Cadet.LLMStats do ) ) + # ADDED: Fetch the cost and token data from the Assessment table + costs = + Repo.one( + from(a in Cadet.Assessments.Assessment, + where: a.id == ^assessment_id and a.course_id == ^course_id, + select: %{ + llm_total_cost: a.llm_total_cost, + llm_total_input_tokens: a.llm_total_input_tokens, + llm_total_output_tokens: a.llm_total_output_tokens, + llm_total_cached_tokens: a.llm_total_cached_tokens + } + ) + ) || %{} + + # Merge the costs into the final map that gets sent to React %{ total_uses: total_uses, unique_submissions: unique_submissions, unique_users: unique_users, - questions: questions + questions: questions, + # Add the cost data (with safe fallbacks if nil) + llm_total_cost: Map.get(costs, :llm_total_cost) || Decimal.new("0.0"), + llm_total_input_tokens: Map.get(costs, :llm_total_input_tokens) || 0, + llm_total_output_tokens: Map.get(costs, :llm_total_output_tokens) || 0, + llm_total_cached_tokens: Map.get(costs, :llm_total_cached_tokens) || 0 } end diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index ae123fcdd..84663ddef 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -227,7 +227,7 @@ defmodule CadetWeb.AICodeAnalysisController do recv_timeout: 60_000 ] }) do - {:ok, %{choices: [%{"message" => %{"content" => content}} | _]}} -> + {:ok, %{choices: [%{"message" => %{"content" => content}} | _], usage: usage}} -> save_comment( answer.id, Enum.at(final_messages, 0).content, @@ -235,6 +235,9 @@ defmodule CadetWeb.AICodeAnalysisController do content ) + # get the tokens consumed and calc cost + Cadet.Assessments.update_llm_usage_and_cost(answer.question.assessment_id, usage) + usage_attrs = %{ course_id: course_id, assessment_id: answer.question.assessment_id, diff --git a/lib/cadet_web/views/assessments_view.ex b/lib/cadet_web/views/assessments_view.ex index 5c22b1f72..215c46c86 100644 --- a/lib/cadet_web/views/assessments_view.ex +++ b/lib/cadet_web/views/assessments_view.ex @@ -36,7 +36,13 @@ defmodule CadetWeb.AssessmentsView do hasTokenCounter: :has_token_counter, isVotingPublished: :is_voting_published, hoursBeforeEarlyXpDecay: & &1.config.hours_before_early_xp_decay, - isLlmGraded: &(&1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""]) + isLlmGraded: &(&1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""]), + llmInputCost: :llm_input_cost, + llmOutputCost: :llm_output_cost, + llmTotalInputTokens: :llm_total_input_tokens, + llmTotalOutputTokens: :llm_total_output_tokens, + llmTotalCachedTokens: :llm_total_cached_tokens, + llmTotalCost: :llm_total_cost }) end diff --git a/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs b/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs new file mode 100644 index 000000000..db94a0ce0 --- /dev/null +++ b/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs @@ -0,0 +1,35 @@ +defmodule Cadet.Repo.Migrations.RepairLlmColumnsOnAssessments do + use Ecto.Migration + + def up do + alter table(:assessments) do + add_if_missing(:llm_input_cost, :decimal, precision: 10, scale: 4) + add_if_missing(:llm_output_cost, :decimal, precision: 10, scale: 4) + add_if_missing(:llm_total_input_tokens, :integer, default: 0) + add_if_missing(:llm_total_output_tokens, :integer, default: 0) + add_if_missing(:llm_total_cached_tokens, :integer, default: 0) + add_if_missing(:llm_total_cost, :decimal, precision: 10, scale: 4, default: 0.0) + end + end + + def down do + alter table(:assessments) do + remove :llm_input_cost + remove :llm_output_cost + remove :llm_total_input_tokens + remove :llm_total_output_tokens + remove :llm_total_cached_tokens + remove :llm_total_cost + end + end + + defp add_if_missing(column, type, opts) do + # Direct check against the database metadata + query = "SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'assessments' AND column_name = '#{column}')" + + case repo().query!(query) do + %{rows: [[false]]} -> add column, type, opts + _ -> :ok + end + end +end diff --git a/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs b/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs new file mode 100644 index 000000000..95338cb91 --- /dev/null +++ b/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs @@ -0,0 +1,7 @@ +defmodule Cadet.Repo.Migrations.RepairLlmColumnsOnAssessments do + use Ecto.Migration + + def change do + + end +end From 0083876a80a3b4b05fd687d3957503e180eee429 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Fri, 20 Mar 2026 21:07:00 +0800 Subject: [PATCH 34/68] chore: save remaining backend assessments and migration files --- lib/cadet/assessments/assessments.ex | 75 ++++++++++--------- ..._add_detailed_llm_costs_to_assessments.exs | 17 +++-- ...4651_repair_llm_columns_on_assessments.exs | 1 - 3 files changed, 50 insertions(+), 43 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index d7dd608ac..36972e3d7 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3631,38 +3631,45 @@ defmodule Cadet.Assessments do Repo.one(query) end - def update_llm_usage_and_cost(assessment_id, usage) do - assessment = Repo.get(Assessment, assessment_id) - - # Try string keys first, fall back to atom keys - prompt = usage["prompt_tokens"] || usage[:prompt_tokens] || 0 - completion = usage["completion_tokens"] || usage[:completion_tokens] || 0 - - # Try to find cached tokens in the nested map - details = usage["prompt_tokens_details"] || usage[:prompt_tokens_details] || %{} - cached = details["cached_tokens"] || details[:cached_tokens] || 0 - - # Logic: Use the keyed-in value if it's > 0, otherwise use SGD defaults - million = Decimal.new(1_000_000) - input_rate_1m = if assessment.llm_input_cost && Decimal.gt?(assessment.llm_input_cost, 0), - do: assessment.llm_input_cost, else: Decimal.new("3.20") - output_rate_1m = if assessment.llm_output_cost && Decimal.gt?(assessment.llm_output_cost, 0), - do: assessment.llm_output_cost, else: Decimal.new("12.80") - - # Math: (Tokens * Rate) / 1,000,000 - new_cost = Decimal.add( - Decimal.div(Decimal.mult(Decimal.new(prompt), input_rate_1m), million), - Decimal.div(Decimal.mult(Decimal.new(completion), output_rate_1m), million) - ) - - # UPDATE CALL - assessment - |> Assessment.changeset(%{ - llm_total_input_tokens: (assessment.llm_total_input_tokens || 0) + prompt, - llm_total_output_tokens: (assessment.llm_total_output_tokens || 0) + completion, - llm_total_cached_tokens: (assessment.llm_total_cached_tokens || 0) + cached, - llm_total_cost: Decimal.add(assessment.llm_total_cost || Decimal.new("0"), new_cost) - }) - |> Repo.update() -end + def update_llm_usage_and_cost(assessment_id, usage) do + assessment = Repo.get(Assessment, assessment_id) + + # Try string keys first, fall back to atom keys + prompt = usage["prompt_tokens"] || usage[:prompt_tokens] || 0 + completion = usage["completion_tokens"] || usage[:completion_tokens] || 0 + + # Try to find cached tokens in the nested map + details = usage["prompt_tokens_details"] || usage[:prompt_tokens_details] || %{} + cached = details["cached_tokens"] || details[:cached_tokens] || 0 + + # Logic: Use the keyed-in value if it's > 0, otherwise use SGD defaults + million = Decimal.new(1_000_000) + + input_rate_1m = + if assessment.llm_input_cost && Decimal.gt?(assessment.llm_input_cost, 0), + do: assessment.llm_input_cost, + else: Decimal.new("3.20") + + output_rate_1m = + if assessment.llm_output_cost && Decimal.gt?(assessment.llm_output_cost, 0), + do: assessment.llm_output_cost, + else: Decimal.new("12.80") + + # Math: (Tokens * Rate) / 1,000,000 + new_cost = + Decimal.add( + Decimal.div(Decimal.mult(Decimal.new(prompt), input_rate_1m), million), + Decimal.div(Decimal.mult(Decimal.new(completion), output_rate_1m), million) + ) + + # UPDATE CALL + assessment + |> Assessment.changeset(%{ + llm_total_input_tokens: (assessment.llm_total_input_tokens || 0) + prompt, + llm_total_output_tokens: (assessment.llm_total_output_tokens || 0) + completion, + llm_total_cached_tokens: (assessment.llm_total_cached_tokens || 0) + cached, + llm_total_cost: Decimal.add(assessment.llm_total_cost || Decimal.new("0"), new_cost) + }) + |> Repo.update() + end end diff --git a/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs b/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs index db94a0ce0..84f026d9c 100644 --- a/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs +++ b/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs @@ -14,21 +14,22 @@ defmodule Cadet.Repo.Migrations.RepairLlmColumnsOnAssessments do def down do alter table(:assessments) do - remove :llm_input_cost - remove :llm_output_cost - remove :llm_total_input_tokens - remove :llm_total_output_tokens - remove :llm_total_cached_tokens - remove :llm_total_cost + remove(:llm_input_cost) + remove(:llm_output_cost) + remove(:llm_total_input_tokens) + remove(:llm_total_output_tokens) + remove(:llm_total_cached_tokens) + remove(:llm_total_cost) end end defp add_if_missing(column, type, opts) do # Direct check against the database metadata - query = "SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'assessments' AND column_name = '#{column}')" + query = + "SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'assessments' AND column_name = '#{column}')" case repo().query!(query) do - %{rows: [[false]]} -> add column, type, opts + %{rows: [[false]]} -> add(column, type, opts) _ -> :ok end end diff --git a/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs b/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs index 95338cb91..e1a201962 100644 --- a/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs +++ b/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs @@ -2,6 +2,5 @@ defmodule Cadet.Repo.Migrations.RepairLlmColumnsOnAssessments do use Ecto.Migration def change do - end end From 6cdbed471a1739deb27f8e78bc6cf0607fdcecc5 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Fri, 20 Mar 2026 21:10:58 +0800 Subject: [PATCH 35/68] refactor: reduce cyclomatic complexity in LLM cost calculator --- lib/cadet/assessments/assessments.ex | 52 ++++++++++++++-------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 36972e3d7..6b66bacbc 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3634,35 +3634,16 @@ defmodule Cadet.Assessments do def update_llm_usage_and_cost(assessment_id, usage) do assessment = Repo.get(Assessment, assessment_id) - # Try string keys first, fall back to atom keys - prompt = usage["prompt_tokens"] || usage[:prompt_tokens] || 0 - completion = usage["completion_tokens"] || usage[:completion_tokens] || 0 + prompt = extract_val(usage, "prompt_tokens", :prompt_tokens, 0) + completion = extract_val(usage, "completion_tokens", :completion_tokens, 0) + details = extract_val(usage, "prompt_tokens_details", :prompt_tokens_details, %{}) + cached = extract_val(details, "cached_tokens", :cached_tokens, 0) - # Try to find cached tokens in the nested map - details = usage["prompt_tokens_details"] || usage[:prompt_tokens_details] || %{} - cached = details["cached_tokens"] || details[:cached_tokens] || 0 + input_rate = get_valid_rate(assessment.llm_input_cost, "3.20") + output_rate = get_valid_rate(assessment.llm_output_cost, "12.80") - # Logic: Use the keyed-in value if it's > 0, otherwise use SGD defaults - million = Decimal.new(1_000_000) - - input_rate_1m = - if assessment.llm_input_cost && Decimal.gt?(assessment.llm_input_cost, 0), - do: assessment.llm_input_cost, - else: Decimal.new("3.20") - - output_rate_1m = - if assessment.llm_output_cost && Decimal.gt?(assessment.llm_output_cost, 0), - do: assessment.llm_output_cost, - else: Decimal.new("12.80") - - # Math: (Tokens * Rate) / 1,000,000 - new_cost = - Decimal.add( - Decimal.div(Decimal.mult(Decimal.new(prompt), input_rate_1m), million), - Decimal.div(Decimal.mult(Decimal.new(completion), output_rate_1m), million) - ) + new_cost = calculate_token_cost(prompt, completion, input_rate, output_rate) - # UPDATE CALL assessment |> Assessment.changeset(%{ llm_total_input_tokens: (assessment.llm_total_input_tokens || 0) + prompt, @@ -3672,4 +3653,23 @@ defmodule Cadet.Assessments do }) |> Repo.update() end + + defp extract_val(map, string_key, atom_key, default) do + Map.get(map, string_key) || Map.get(map, atom_key) || default + end + + defp get_valid_rate(rate, default_rate) do + if rate && Decimal.gt?(rate, 0) do + rate + else + Decimal.new(default_rate) + end + end + + defp calculate_token_cost(prompt, completion, input_rate, output_rate) do + million = Decimal.new(1_000_000) + in_cost = Decimal.div(Decimal.mult(Decimal.new(prompt), input_rate), million) + out_cost = Decimal.div(Decimal.mult(Decimal.new(completion), output_rate), million) + Decimal.add(in_cost, out_cost) + end end From a9dd2f31f86e127966692f97fc80713ced4c7f0d Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 21 Mar 2026 11:49:24 +0800 Subject: [PATCH 36/68] pass testcases for llmcost calculations --- ..._add_detailed_llm_costs_to_assessments.exs | 35 +++++++------------ .../assessments/assessment_factory.ex | 9 ++++- test/test_helper.exs | 19 ++++++++++ 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs b/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs index 84f026d9c..5449c7cc9 100644 --- a/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs +++ b/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs @@ -3,34 +3,23 @@ defmodule Cadet.Repo.Migrations.RepairLlmColumnsOnAssessments do def up do alter table(:assessments) do - add_if_missing(:llm_input_cost, :decimal, precision: 10, scale: 4) - add_if_missing(:llm_output_cost, :decimal, precision: 10, scale: 4) - add_if_missing(:llm_total_input_tokens, :integer, default: 0) - add_if_missing(:llm_total_output_tokens, :integer, default: 0) - add_if_missing(:llm_total_cached_tokens, :integer, default: 0) - add_if_missing(:llm_total_cost, :decimal, precision: 10, scale: 4, default: 0.0) + add_if_not_exists(:llm_input_cost, :decimal, precision: 10, scale: 4) + add_if_not_exists(:llm_output_cost, :decimal, precision: 10, scale: 4) + add_if_not_exists(:llm_total_input_tokens, :integer, default: 0) + add_if_not_exists(:llm_total_output_tokens, :integer, default: 0) + add_if_not_exists(:llm_total_cached_tokens, :integer, default: 0) + add_if_not_exists(:llm_total_cost, :decimal, precision: 10, scale: 4, default: 0.0) end end def down do alter table(:assessments) do - remove(:llm_input_cost) - remove(:llm_output_cost) - remove(:llm_total_input_tokens) - remove(:llm_total_output_tokens) - remove(:llm_total_cached_tokens) - remove(:llm_total_cost) - end - end - - defp add_if_missing(column, type, opts) do - # Direct check against the database metadata - query = - "SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'assessments' AND column_name = '#{column}')" - - case repo().query!(query) do - %{rows: [[false]]} -> add(column, type, opts) - _ -> :ok + remove_if_exists(:llm_input_cost) + remove_if_exists(:llm_output_cost) + remove_if_exists(:llm_total_input_tokens) + remove_if_exists(:llm_total_output_tokens) + remove_if_exists(:llm_total_cached_tokens) + remove_if_exists(:llm_total_cost) end end end diff --git a/test/factories/assessments/assessment_factory.ex b/test/factories/assessments/assessment_factory.ex index 5dba2955d..3b97f0a55 100644 --- a/test/factories/assessments/assessment_factory.ex +++ b/test/factories/assessments/assessment_factory.ex @@ -40,7 +40,14 @@ defmodule Cadet.Assessments.AssessmentFactory do close_at: Timex.shift(Timex.now(), days: Enum.random(1..30)), is_published: false, max_team_size: 1, - llm_assessment_prompt: nil + llm_assessment_prompt: nil, + + llm_input_cost: Decimal.new("3.20"), + llm_output_cost: Decimal.new("12.80"), + llm_total_input_tokens: 0, + llm_total_output_tokens: 0, + llm_total_cached_tokens: 0, + llm_total_cost: Decimal.new("0.0") } end end diff --git a/test/test_helper.exs b/test/test_helper.exs index 30e668048..94e5c4667 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -5,4 +5,23 @@ System.put_env("LEADER", "1") ExUnit.start() Faker.start() +# Ensure test database exists and migrations are run +_ = Ecto.Adapters.Postgres.ensure_all_started(Cadet.Repo, :temporary) +{:ok, _pid} = Cadet.Repo.start_link() + +case Ecto.Adapters.Postgres.storage_down(Cadet.Repo) do + :ok -> :ok + {:error, :already_down} -> :ok + {:error, _} -> :ok +end + +case Ecto.Adapters.Postgres.storage_up(Cadet.Repo) do + :ok -> :ok + {:error, :already_up} -> :ok + {:error, _} -> :ok +end + +# Run all pending migrations +:ok = Ecto.Migrator.run(Cadet.Repo, :up, all: true) + Ecto.Adapters.SQL.Sandbox.mode(Cadet.Repo, :manual) From 425fd6328c6194b063755931bffbaf837cdccc10 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 21 Mar 2026 11:52:38 +0800 Subject: [PATCH 37/68] pass testcases for llmcost calculations --- test/factories/assessments/assessment_factory.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/test/factories/assessments/assessment_factory.ex b/test/factories/assessments/assessment_factory.ex index 3b97f0a55..19c893580 100644 --- a/test/factories/assessments/assessment_factory.ex +++ b/test/factories/assessments/assessment_factory.ex @@ -41,7 +41,6 @@ defmodule Cadet.Assessments.AssessmentFactory do is_published: false, max_team_size: 1, llm_assessment_prompt: nil, - llm_input_cost: Decimal.new("3.20"), llm_output_cost: Decimal.new("12.80"), llm_total_input_tokens: 0, From fa939fbf41f620d50ef38f7b33e099a9b9a04fe4 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 21 Mar 2026 11:56:29 +0800 Subject: [PATCH 38/68] pass testcases for llmcost calculations --- .../controllers/generate_ai_comments.ex | 112 ++++++++++-------- 1 file changed, 63 insertions(+), 49 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 84663ddef..103d9e384 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -227,58 +227,72 @@ defmodule CadetWeb.AICodeAnalysisController do recv_timeout: 60_000 ] }) do - {:ok, %{choices: [%{"message" => %{"content" => content}} | _], usage: usage}} -> - save_comment( - answer.id, - Enum.at(final_messages, 0).content, - Enum.at(final_messages, 1).content, - content - ) - - # get the tokens consumed and calc cost - Cadet.Assessments.update_llm_usage_and_cost(answer.question.assessment_id, usage) - - usage_attrs = %{ - course_id: course_id, - assessment_id: answer.question.assessment_id, - question_id: answer.question_id, - answer_id: answer.id, - submission_id: answer.submission_id, - user_id: conn.assigns.course_reg.user_id - } - - # Log LLM usage for statistics (non-blocking for response generation) - case LLMStats.log_usage(usage_attrs) do - {:ok, _usage_log} -> - :ok - - {:error, changeset} -> - Logger.error( - "Failed to log LLM usage to database: #{inspect(changeset.errors)} attrs=#{inspect(usage_attrs)}" + {:ok, response} -> + # Handle cases where API may or may not return usage field + case response do + %{choices: [%{"message" => %{"content" => content}} | _]} -> + save_comment( + answer.id, + Enum.at(final_messages, 0).content, + Enum.at(final_messages, 1).content, + content ) - end - comments_list = String.split(content, "|||") - - filtered_comments = - Enum.filter(comments_list, fn comment -> - String.trim(comment) != "" - end) - - json(conn, %{"comments" => filtered_comments}) - - {:ok, other} -> - save_comment( - answer.id, - Enum.at(final_messages, 0).content, - Enum.at(final_messages, 1).content, - Jason.encode!(other), - "Unexpected JSON shape" - ) + # Optionally update cost tracking if usage data is available + case Map.get(response, :usage) do + nil -> + Logger.warning("LLM API response missing usage field for answer_id=#{answer.id}") + + usage -> + # get the tokens consumed and calc cost + Cadet.Assessments.update_llm_usage_and_cost( + answer.question.assessment_id, + usage + ) + end + + usage_attrs = %{ + course_id: course_id, + assessment_id: answer.question.assessment_id, + question_id: answer.question_id, + answer_id: answer.id, + submission_id: answer.submission_id, + user_id: conn.assigns.course_reg.user_id + } + + # Log LLM usage for statistics (non-blocking for response generation) + case LLMStats.log_usage(usage_attrs) do + {:ok, _usage_log} -> + :ok + + {:error, changeset} -> + Logger.error( + "Failed to log LLM usage to database: #{inspect(changeset.errors)} attrs=#{inspect(usage_attrs)}" + ) + end + + comments_list = String.split(content, "|||") + + filtered_comments = + Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) + + json(conn, %{"comments" => filtered_comments}) + + _ -> + save_comment( + answer.id, + Enum.at(final_messages, 0).content, + Enum.at(final_messages, 1).content, + Jason.encode!(response), + "Unexpected JSON shape" + ) - conn - |> put_status(:bad_gateway) - |> text("Unexpected response format from LLM") + conn + |> put_status(:bad_gateway) + |> text("Unexpected response format from LLM") + end {:error, reason} -> save_comment( From 4dd4ed3b69a547d045b0659f5945c5b5e0b4e4f6 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 21 Mar 2026 12:00:25 +0800 Subject: [PATCH 39/68] pass testcases for llmcost calculations --- lib/cadet/assessments/assessments.ex | 27 ++++++++++++------- ..._add_detailed_llm_costs_to_assessments.exs | 12 ++++----- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 6b66bacbc..22e0a0133 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3632,26 +3632,33 @@ defmodule Cadet.Assessments do end def update_llm_usage_and_cost(assessment_id, usage) do - assessment = Repo.get(Assessment, assessment_id) - prompt = extract_val(usage, "prompt_tokens", :prompt_tokens, 0) completion = extract_val(usage, "completion_tokens", :completion_tokens, 0) details = extract_val(usage, "prompt_tokens_details", :prompt_tokens_details, %{}) cached = extract_val(details, "cached_tokens", :cached_tokens, 0) + # Fetch assessment to get cost rates + assessment = Repo.get(Assessment, assessment_id) + input_rate = get_valid_rate(assessment.llm_input_cost, "3.20") output_rate = get_valid_rate(assessment.llm_output_cost, "12.80") new_cost = calculate_token_cost(prompt, completion, input_rate, output_rate) - assessment - |> Assessment.changeset(%{ - llm_total_input_tokens: (assessment.llm_total_input_tokens || 0) + prompt, - llm_total_output_tokens: (assessment.llm_total_output_tokens || 0) + completion, - llm_total_cached_tokens: (assessment.llm_total_cached_tokens || 0) + cached, - llm_total_cost: Decimal.add(assessment.llm_total_cost || Decimal.new("0"), new_cost) - }) - |> Repo.update() + # Atomic database-level updates to prevent race conditions + # All increments happen in a single transaction at the database level + Repo.update_all( + from(a in Assessment, where: a.id == ^assessment_id), + set: [ + llm_total_input_tokens: fragment("COALESCE(llm_total_input_tokens, 0) + ?", ^prompt), + llm_total_output_tokens: + fragment("COALESCE(llm_total_output_tokens, 0) + ?", ^completion), + llm_total_cached_tokens: fragment("COALESCE(llm_total_cached_tokens, 0) + ?", ^cached), + llm_total_cost: fragment("COALESCE(llm_total_cost, 0) + ?", ^new_cost) + ] + ) + + {:ok, nil} end defp extract_val(map, string_key, atom_key, default) do diff --git a/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs b/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs index 5449c7cc9..0d1c51551 100644 --- a/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs +++ b/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs @@ -14,12 +14,12 @@ defmodule Cadet.Repo.Migrations.RepairLlmColumnsOnAssessments do def down do alter table(:assessments) do - remove_if_exists(:llm_input_cost) - remove_if_exists(:llm_output_cost) - remove_if_exists(:llm_total_input_tokens) - remove_if_exists(:llm_total_output_tokens) - remove_if_exists(:llm_total_cached_tokens) - remove_if_exists(:llm_total_cost) + remove(:llm_input_cost) + remove(:llm_output_cost) + remove(:llm_total_input_tokens) + remove(:llm_total_output_tokens) + remove(:llm_total_cached_tokens) + remove(:llm_total_cost) end end end From 261e7993e571a0ce9eeb196b76d15319e641b266 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 21 Mar 2026 15:23:22 +0800 Subject: [PATCH 40/68] pass testcases for llmcost calculations --- lib/cadet/assessments/assessments.ex | 17 +++++++-------- ..._add_detailed_llm_costs_to_assessments.exs | 16 +++++++------- ...4651_repair_llm_columns_on_assessments.exs | 21 ++++++++++++++++++- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 22e0a0133..360716360 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3647,16 +3647,13 @@ defmodule Cadet.Assessments do # Atomic database-level updates to prevent race conditions # All increments happen in a single transaction at the database level - Repo.update_all( - from(a in Assessment, where: a.id == ^assessment_id), - set: [ - llm_total_input_tokens: fragment("COALESCE(llm_total_input_tokens, 0) + ?", ^prompt), - llm_total_output_tokens: - fragment("COALESCE(llm_total_output_tokens, 0) + ?", ^completion), - llm_total_cached_tokens: fragment("COALESCE(llm_total_cached_tokens, 0) + ?", ^cached), - llm_total_cost: fragment("COALESCE(llm_total_cost, 0) + ?", ^new_cost) - ] - ) + query = from(a in Assessment, where: a.id == ^assessment_id, update: [set: [ + llm_total_input_tokens: fragment("COALESCE(llm_total_input_tokens, 0) + ?", ^prompt), + llm_total_output_tokens: fragment("COALESCE(llm_total_output_tokens, 0) + ?", ^completion), + llm_total_cached_tokens: fragment("COALESCE(llm_total_cached_tokens, 0) + ?", ^cached), + llm_total_cost: fragment("COALESCE(llm_total_cost, 0) + ?", ^new_cost) + ]]) + Repo.update_all(query, []) {:ok, nil} end diff --git a/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs b/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs index 0d1c51551..9b25c6316 100644 --- a/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs +++ b/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs @@ -3,8 +3,8 @@ defmodule Cadet.Repo.Migrations.RepairLlmColumnsOnAssessments do def up do alter table(:assessments) do - add_if_not_exists(:llm_input_cost, :decimal, precision: 10, scale: 4) - add_if_not_exists(:llm_output_cost, :decimal, precision: 10, scale: 4) + add_if_not_exists(:llm_input_cost, :decimal, precision: 10, scale: 4, default: 3.20) + add_if_not_exists(:llm_output_cost, :decimal, precision: 10, scale: 4, default: 12.80) add_if_not_exists(:llm_total_input_tokens, :integer, default: 0) add_if_not_exists(:llm_total_output_tokens, :integer, default: 0) add_if_not_exists(:llm_total_cached_tokens, :integer, default: 0) @@ -14,12 +14,12 @@ defmodule Cadet.Repo.Migrations.RepairLlmColumnsOnAssessments do def down do alter table(:assessments) do - remove(:llm_input_cost) - remove(:llm_output_cost) - remove(:llm_total_input_tokens) - remove(:llm_total_output_tokens) - remove(:llm_total_cached_tokens) - remove(:llm_total_cost) + remove_if_exists(:llm_input_cost, :decimal) + remove_if_exists(:llm_output_cost, :decimal) + remove_if_exists(:llm_total_input_tokens, :integer) + remove_if_exists(:llm_total_output_tokens, :integer) + remove_if_exists(:llm_total_cached_tokens, :integer) + remove_if_exists(:llm_total_cost, :decimal) end end end diff --git a/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs b/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs index e1a201962..371fca7b0 100644 --- a/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs +++ b/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs @@ -1,6 +1,25 @@ defmodule Cadet.Repo.Migrations.RepairLlmColumnsOnAssessments do use Ecto.Migration - def change do + def up do + alter table(:assessments) do + add_if_not_exists :llm_input_cost, :decimal, precision: 10, scale: 4, default: 3.20 + add_if_not_exists :llm_output_cost, :decimal, precision: 10, scale: 4, default: 12.80 + add_if_not_exists :llm_total_input_tokens, :integer, default: 0 + add_if_not_exists :llm_total_output_tokens, :integer, default: 0 + add_if_not_exists :llm_total_cached_tokens, :integer, default: 0 + add_if_not_exists :llm_total_cost, :decimal, precision: 10, scale: 4, default: 0.0 + end + end + + def down do + alter table(:assessments) do + remove_if_exists :llm_input_cost, :decimal + remove_if_exists :llm_output_cost, :decimal + remove_if_exists :llm_total_input_tokens, :integer + remove_if_exists :llm_total_output_tokens, :integer + remove_if_exists :llm_total_cached_tokens, :integer + remove_if_exists :llm_total_cost, :decimal + end end end From 17670d5ea9249909d7ad62a404293748bc5835a9 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Thu, 26 Mar 2026 16:27:20 +0800 Subject: [PATCH 41/68] pass testcases for llmcost calculations --- lib/cadet/assessments/assessments.ex | 33 ++++++++++++--------- test/cadet/assessments/assessments_test.exs | 26 ++++++++++++++++ 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 360716360..c80662290 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3638,24 +3638,29 @@ defmodule Cadet.Assessments do cached = extract_val(details, "cached_tokens", :cached_tokens, 0) # Fetch assessment to get cost rates - assessment = Repo.get(Assessment, assessment_id) + case Repo.get(Assessment, assessment_id) do + nil -> + Logger.error("Assessment not found when updating LLM usage and cost: #{assessment_id}") + {:error, :not_found} - input_rate = get_valid_rate(assessment.llm_input_cost, "3.20") - output_rate = get_valid_rate(assessment.llm_output_cost, "12.80") + assessment -> + input_rate = get_valid_rate(assessment.llm_input_cost, "3.20") + output_rate = get_valid_rate(assessment.llm_output_cost, "12.80") - new_cost = calculate_token_cost(prompt, completion, input_rate, output_rate) + new_cost = calculate_token_cost(prompt, completion, input_rate, output_rate) - # Atomic database-level updates to prevent race conditions - # All increments happen in a single transaction at the database level - query = from(a in Assessment, where: a.id == ^assessment_id, update: [set: [ - llm_total_input_tokens: fragment("COALESCE(llm_total_input_tokens, 0) + ?", ^prompt), - llm_total_output_tokens: fragment("COALESCE(llm_total_output_tokens, 0) + ?", ^completion), - llm_total_cached_tokens: fragment("COALESCE(llm_total_cached_tokens, 0) + ?", ^cached), - llm_total_cost: fragment("COALESCE(llm_total_cost, 0) + ?", ^new_cost) - ]]) - Repo.update_all(query, []) + # Atomic database-level updates to prevent race conditions + # All increments happen in a single transaction at the database level + query = from(a in Assessment, where: a.id == ^assessment_id, update: [set: [ + llm_total_input_tokens: fragment("COALESCE(llm_total_input_tokens, 0) + ?", ^prompt), + llm_total_output_tokens: fragment("COALESCE(llm_total_output_tokens, 0) + ?", ^completion), + llm_total_cached_tokens: fragment("COALESCE(llm_total_cached_tokens, 0) + ?", ^cached), + llm_total_cost: fragment("COALESCE(llm_total_cost, 0) + ?", ^new_cost) + ]]) - {:ok, nil} + Repo.update_all(query, []) + {:ok, nil} + end end defp extract_val(map, string_key, atom_key, default) do diff --git a/test/cadet/assessments/assessments_test.exs b/test/cadet/assessments/assessments_test.exs index cd86b020a..c3a015556 100644 --- a/test/cadet/assessments/assessments_test.exs +++ b/test/cadet/assessments/assessments_test.exs @@ -158,6 +158,32 @@ defmodule Cadet.AssessmentsTest do ) end + test "update_llm_usage_and_cost handles nonexistent assessment gracefully" do + usage = %{"prompt_tokens" => 10, "completion_tokens" => 20, "prompt_tokens_details" => %{"cached_tokens" => 5}} + + assert {:error, :not_found} = Assessments.update_llm_usage_and_cost(-1, usage) + end + + test "update_llm_usage_and_cost explicitly preserves nil assessment path" do + usage = %{"prompt_tokens" => 1, "completion_tokens" => 1, "prompt_tokens_details" => %{"cached_tokens" => 0}} + + assert {:error, :not_found} = Assessments.update_llm_usage_and_cost(-999_999, usage) + end + + test "update_llm_usage_and_cost increments LLM totals for existing assessment" do + assessment = insert(:assessment, %{llm_total_input_tokens: 0, llm_total_output_tokens: 0, llm_total_cached_tokens: 0, llm_total_cost: Decimal.new("0.0")}) + usage = %{"prompt_tokens" => 10, "completion_tokens" => 20, "prompt_tokens_details" => %{"cached_tokens" => 5}} + + assert {:ok, nil} = Assessments.update_llm_usage_and_cost(assessment.id, usage) + + updated = Repo.get(Assessment, assessment.id) + + assert updated.llm_total_input_tokens == 10 + assert updated.llm_total_output_tokens == 20 + assert updated.llm_total_cached_tokens == 5 + assert Decimal.cmp(updated.llm_total_cost, Decimal.new("0.000288")) == :eq + end + test "force update assessment with invalid params" do course = insert(:course) config = insert(:assessment_config, %{course: course}) From cadf106ce107d752740ce86139cf500a2d9308ce Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Thu, 26 Mar 2026 16:30:18 +0800 Subject: [PATCH 42/68] pass testcases for llmcost calculations --- lib/cadet/assessments/assessments.ex | 21 ++++++++++----- ...4651_repair_llm_columns_on_assessments.exs | 24 ++++++++--------- test/cadet/assessments/assessments_test.exs | 27 ++++++++++++++++--- 3 files changed, 50 insertions(+), 22 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index c80662290..bdd79e097 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3651,12 +3651,21 @@ defmodule Cadet.Assessments do # Atomic database-level updates to prevent race conditions # All increments happen in a single transaction at the database level - query = from(a in Assessment, where: a.id == ^assessment_id, update: [set: [ - llm_total_input_tokens: fragment("COALESCE(llm_total_input_tokens, 0) + ?", ^prompt), - llm_total_output_tokens: fragment("COALESCE(llm_total_output_tokens, 0) + ?", ^completion), - llm_total_cached_tokens: fragment("COALESCE(llm_total_cached_tokens, 0) + ?", ^cached), - llm_total_cost: fragment("COALESCE(llm_total_cost, 0) + ?", ^new_cost) - ]]) + query = + from(a in Assessment, + where: a.id == ^assessment_id, + update: [ + set: [ + llm_total_input_tokens: + fragment("COALESCE(llm_total_input_tokens, 0) + ?", ^prompt), + llm_total_output_tokens: + fragment("COALESCE(llm_total_output_tokens, 0) + ?", ^completion), + llm_total_cached_tokens: + fragment("COALESCE(llm_total_cached_tokens, 0) + ?", ^cached), + llm_total_cost: fragment("COALESCE(llm_total_cost, 0) + ?", ^new_cost) + ] + ] + ) Repo.update_all(query, []) {:ok, nil} diff --git a/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs b/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs index 371fca7b0..9b25c6316 100644 --- a/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs +++ b/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs @@ -3,23 +3,23 @@ defmodule Cadet.Repo.Migrations.RepairLlmColumnsOnAssessments do def up do alter table(:assessments) do - add_if_not_exists :llm_input_cost, :decimal, precision: 10, scale: 4, default: 3.20 - add_if_not_exists :llm_output_cost, :decimal, precision: 10, scale: 4, default: 12.80 - add_if_not_exists :llm_total_input_tokens, :integer, default: 0 - add_if_not_exists :llm_total_output_tokens, :integer, default: 0 - add_if_not_exists :llm_total_cached_tokens, :integer, default: 0 - add_if_not_exists :llm_total_cost, :decimal, precision: 10, scale: 4, default: 0.0 + add_if_not_exists(:llm_input_cost, :decimal, precision: 10, scale: 4, default: 3.20) + add_if_not_exists(:llm_output_cost, :decimal, precision: 10, scale: 4, default: 12.80) + add_if_not_exists(:llm_total_input_tokens, :integer, default: 0) + add_if_not_exists(:llm_total_output_tokens, :integer, default: 0) + add_if_not_exists(:llm_total_cached_tokens, :integer, default: 0) + add_if_not_exists(:llm_total_cost, :decimal, precision: 10, scale: 4, default: 0.0) end end def down do alter table(:assessments) do - remove_if_exists :llm_input_cost, :decimal - remove_if_exists :llm_output_cost, :decimal - remove_if_exists :llm_total_input_tokens, :integer - remove_if_exists :llm_total_output_tokens, :integer - remove_if_exists :llm_total_cached_tokens, :integer - remove_if_exists :llm_total_cost, :decimal + remove_if_exists(:llm_input_cost, :decimal) + remove_if_exists(:llm_output_cost, :decimal) + remove_if_exists(:llm_total_input_tokens, :integer) + remove_if_exists(:llm_total_output_tokens, :integer) + remove_if_exists(:llm_total_cached_tokens, :integer) + remove_if_exists(:llm_total_cost, :decimal) end end end diff --git a/test/cadet/assessments/assessments_test.exs b/test/cadet/assessments/assessments_test.exs index c3a015556..202275047 100644 --- a/test/cadet/assessments/assessments_test.exs +++ b/test/cadet/assessments/assessments_test.exs @@ -159,20 +159,39 @@ defmodule Cadet.AssessmentsTest do end test "update_llm_usage_and_cost handles nonexistent assessment gracefully" do - usage = %{"prompt_tokens" => 10, "completion_tokens" => 20, "prompt_tokens_details" => %{"cached_tokens" => 5}} + usage = %{ + "prompt_tokens" => 10, + "completion_tokens" => 20, + "prompt_tokens_details" => %{"cached_tokens" => 5} + } assert {:error, :not_found} = Assessments.update_llm_usage_and_cost(-1, usage) end test "update_llm_usage_and_cost explicitly preserves nil assessment path" do - usage = %{"prompt_tokens" => 1, "completion_tokens" => 1, "prompt_tokens_details" => %{"cached_tokens" => 0}} + usage = %{ + "prompt_tokens" => 1, + "completion_tokens" => 1, + "prompt_tokens_details" => %{"cached_tokens" => 0} + } assert {:error, :not_found} = Assessments.update_llm_usage_and_cost(-999_999, usage) end test "update_llm_usage_and_cost increments LLM totals for existing assessment" do - assessment = insert(:assessment, %{llm_total_input_tokens: 0, llm_total_output_tokens: 0, llm_total_cached_tokens: 0, llm_total_cost: Decimal.new("0.0")}) - usage = %{"prompt_tokens" => 10, "completion_tokens" => 20, "prompt_tokens_details" => %{"cached_tokens" => 5}} + assessment = + insert(:assessment, %{ + llm_total_input_tokens: 0, + llm_total_output_tokens: 0, + llm_total_cached_tokens: 0, + llm_total_cost: Decimal.new("0.0") + }) + + usage = %{ + "prompt_tokens" => 10, + "completion_tokens" => 20, + "prompt_tokens_details" => %{"cached_tokens" => 5} + } assert {:ok, nil} = Assessments.update_llm_usage_and_cost(assessment.id, usage) From 1c7fa0914c9516abb98486c3655c67edbf47782e Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Thu, 26 Mar 2026 16:41:40 +0800 Subject: [PATCH 43/68] pass testcases for llmcost calculations --- config/runtime.exs | 9 +++ lib/cadet/assessments/assessments.ex | 97 +++++++++++++++------------- lib/cadet_web/endpoint.ex | 15 ----- test/test_helper.exs | 6 +- 4 files changed, 67 insertions(+), 60 deletions(-) create mode 100644 config/runtime.exs diff --git a/config/runtime.exs b/config/runtime.exs new file mode 100644 index 000000000..49082da62 --- /dev/null +++ b/config/runtime.exs @@ -0,0 +1,9 @@ +import Config + +# This file is executed after the code compilation on all environments. +# It contains runtime configuration that's evaluated when the system starts. + +# Configure the port from environment variable if set +if port = System.get_env("PORT") do + config :cadet, CadetWeb.Endpoint, http: [:inet6, port: String.to_integer(port)] +end diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index bdd79e097..4806a8ee8 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -1549,57 +1549,66 @@ defmodule Cadet.Assessments do delete_comments_for_answers(answer_ids) {:ok, nil} end) - |> Repo.transaction() - case submission.student_id do - # Team submission, handle notifications for team members - nil -> - Logger.info("Handling unsubmit notifications for team submission #{submission.id}") - team = Repo.get(Team, submission.team_id) - - query = - from(t in Team, - join: tm in TeamMember, - on: t.id == tm.team_id, - join: cr in CourseRegistration, - on: tm.student_id == cr.id, - where: t.id == ^team.id, - select: cr.id - ) + transaction_result = Repo.transaction() + + case transaction_result do + {:ok, _result} -> + Logger.info("Successfully unsubmitting submission #{submission_id}") + + case submission.student_id do + # Team submission, handle notifications for team members + nil -> + Logger.info("Handling unsubmit notifications for team submission #{submission.id}") + team = Repo.get(Team, submission.team_id) + + query = + from(t in Team, + join: tm in TeamMember, + on: t.id == tm.team_id, + join: cr in CourseRegistration, + on: tm.student_id == cr.id, + where: t.id == ^team.id, + select: cr.id + ) + + team_members = Repo.all(query) + + Enum.each(team_members, fn tm_id -> + Logger.info("Sending unsubmit notification to team member #{tm_id}") + + Notifications.handle_unsubmit_notifications( + submission.assessment.id, + Repo.get(CourseRegistration, tm_id) + ) + end) + + student_id -> + Logger.info( + "Handling unsubmit notifications for individual submission #{submission.id}" + ) - team_members = Repo.all(query) + Notifications.handle_unsubmit_notifications( + submission.assessment.id, + Repo.get(CourseRegistration, student_id) + ) + end - Enum.each(team_members, fn tm_id -> - Logger.info("Sending unsubmit notification to team member #{tm_id}") + Logger.info("Removing grading notifications for submission #{submission.id}") - Notifications.handle_unsubmit_notifications( - submission.assessment.id, - Repo.get(CourseRegistration, tm_id) - ) - end) + # Remove grading notifications for submissions + Notification + |> where(submission_id: ^submission_id, type: :submitted) + |> select([n], n.id) + |> Repo.all() + |> Notifications.acknowledge(cr) - student_id -> - Logger.info( - "Handling unsubmit notifications for individual submission #{submission.id}" - ) + {:ok, nil} - Notifications.handle_unsubmit_notifications( - submission.assessment.id, - Repo.get(CourseRegistration, student_id) - ) + {:error, _failed_operation, failed_value, _changes_so_far} -> + Logger.error("Failed to unsubmit submission #{submission_id}: #{inspect(failed_value)}") + {:error, {:internal_server_error, "Failed to unsubmit submission"}} end - - Logger.info("Removing grading notifications for submission #{submission.id}") - - # Remove grading notifications for submissions - Notification - |> where(submission_id: ^submission_id, type: :submitted) - |> select([n], n.id) - |> Repo.all() - |> Notifications.acknowledge(cr) - - Logger.info("Successfully unsubmitting submission #{submission_id}") - {:ok, nil} else {:submission_found?, false} -> Logger.error("Submission #{submission_id} not found") diff --git a/lib/cadet_web/endpoint.ex b/lib/cadet_web/endpoint.ex index 57f801b83..be96645f8 100644 --- a/lib/cadet_web/endpoint.ex +++ b/lib/cadet_web/endpoint.ex @@ -56,19 +56,4 @@ defmodule CadetWeb.Endpoint do ) plug(CadetWeb.Router) - - @doc """ - Callback invoked for dynamically configuring the endpoint. - - It receives the endpoint configuration and checks if - configuration should be loaded from the system environment. - """ - def init(_key, config) do - if config[:load_from_system_env] do - port = System.get_env("PORT") || raise "expected the PORT environment variable to be set" - {:ok, Keyword.put(config, :http, [:inet6, port: port] ++ (config[:http] || []))} - else - {:ok, config} - end - end end diff --git a/test/test_helper.exs b/test/test_helper.exs index 94e5c4667..6004b70aa 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -7,7 +7,11 @@ Faker.start() # Ensure test database exists and migrations are run _ = Ecto.Adapters.Postgres.ensure_all_started(Cadet.Repo, :temporary) -{:ok, _pid} = Cadet.Repo.start_link() + +case Cadet.Repo.start_link() do + {:ok, _pid} -> :ok + {:error, {:already_started, _pid}} -> :ok +end case Ecto.Adapters.Postgres.storage_down(Cadet.Repo) do :ok -> :ok From 970cda29bb15566f806906ca4471da7c6f8c4b18 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Thu, 26 Mar 2026 16:54:17 +0800 Subject: [PATCH 44/68] pass testcases for llmcost calculations --- lib/cadet/assessments/assessments.ex | 199 +++++++++--------- lib/cadet/jobs/autograder/grading_job.ex | 2 +- .../jobs/autograder/result_store_worker.ex | 2 +- lib/cadet/workers/NotificationWorker.ex | 18 +- .../admin_assessments_controller.ex | 4 +- .../controllers/answer_controller.ex | 4 +- .../controllers/assessments_controller.ex | 2 +- test/cadet/assessments/assessments_test.exs | 8 +- 8 files changed, 125 insertions(+), 114 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 4806a8ee8..74ecd903c 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -604,7 +604,7 @@ defmodule Cadet.Assessments do ) end - defp is_voting_assigned(assessment_ids) do + defp voting_assigned?(assessment_ids) do Logger.debug("Checking if voting is assigned for assessment IDs: #{inspect(assessment_ids)}") voting_assigned_question_ids = @@ -639,7 +639,7 @@ defmodule Cadet.Assessments do is_voting_assigned_map = assessments |> Enum.map(& &1.id) - |> is_voting_assigned() + |> voting_assigned?() Enum.map(assessments, fn a -> a = Map.put(a, :is_voting_published, Map.get(is_voting_assigned_map, a.id, false)) @@ -910,7 +910,7 @@ defmodule Cadet.Assessments do ) if is_reassigning_voting do - if is_voting_published(assessment_id) do + if voting_published?(assessment_id) do Logger.info("Deleting existing submissions for assessment #{assessment_id}") Submission @@ -955,7 +955,7 @@ defmodule Cadet.Assessments do end end - defp is_voting_published(assessment_id) do + defp voting_published?(assessment_id) do Logger.info("Checking if voting is published for assessment #{assessment_id}") voting_assigned_question_ids = @@ -1467,6 +1467,16 @@ defmodule Cadet.Assessments do when is_ecto_id(submission_id) do Logger.info("Unsubmitting submission #{submission_id} for user #{course_reg_id}") + case validate_unsubmit_submission(submission_id, cr) do + {:ok, submission} -> + perform_unsubmit_transaction(submission, submission_id, cr) + + {:error, reason} -> + reason + end + end + + defp validate_unsubmit_submission(submission_id, cr = %CourseRegistration{role: role}) do submission = Submission |> join(:inner, [s], a in assoc(s, :assessment)) @@ -1474,11 +1484,10 @@ defmodule Cadet.Assessments do |> Repo.get(submission_id) # allows staff to unsubmit own assessment - bypass = role in @bypass_closed_roles and submission.student_id == course_reg_id - Logger.info("Bypass restrictions: #{bypass}") + bypass = role in @bypass_closed_roles and submission.student_id == cr.id with {:submission_found?, true} <- {:submission_found?, is_map(submission)}, - {:is_open?, true} <- {:is_open?, bypass or is_open?(submission.assessment)}, + {:is_open?, true} <- {:is_open?, bypass or open?(submission.assessment)}, {:status, :submitted} <- {:status, submission.status}, {:allowed_to_unsubmit?, true} <- {:allowed_to_unsubmit?, @@ -1486,8 +1495,42 @@ defmodule Cadet.Assessments do Cadet.Accounts.Query.avenger_of?(cr, submission.student_id)}, {:is_grading_published?, false} <- {:is_grading_published?, submission.is_grading_published} do - Logger.info("All checks passed for unsubmitting submission #{submission_id}") + {:ok, submission} + else + {:submission_found?, false} -> + Logger.error("Submission #{submission_id} not found") + {:error, {:not_found, "Submission not found"}} + + {:is_open?, false} -> + Logger.error("Assessment for submission #{submission_id} is not open") + {:error, {:forbidden, "Assessment not open"}} + {:status, :attempting} -> + Logger.error("Submission #{submission_id} is still attempting") + {:error, {:bad_request, "Some questions have not been attempted"}} + + {:status, :attempted} -> + Logger.error("Submission #{submission_id} has already been attempted") + {:error, {:bad_request, "Assessment has not been submitted"}} + + {:allowed_to_unsubmit?, false} -> + Logger.error("User #{cr.id} is not allowed to unsubmit submission #{submission_id}") + {:error, {:forbidden, "Only Avenger of student or Admin is permitted to unsubmit"}} + + {:is_grading_published?, true} -> + Logger.error("Grading for submission #{submission_id} has already been published") + {:error, {:forbidden, "Grading has not been unpublished"}} + + _ -> + Logger.error("An unknown error occurred while unsubmitting submission #{submission_id}") + {:error, {:internal_server_error, "Please try again later."}} + end + end + + defp perform_unsubmit_transaction(submission, submission_id, cr) do + Logger.info("All checks passed for unsubmitting submission #{submission_id}") + + multi = Multi.new() |> Multi.run( :rollback_submission, @@ -1498,7 +1541,7 @@ defmodule Cadet.Assessments do |> Submission.changeset(%{ status: :attempted, xp_bonus: 0, - unsubmitted_by_id: course_reg_id, + unsubmitted_by_id: cr.id, unsubmitted_at: Timex.now() }) |> Repo.update() @@ -1550,96 +1593,64 @@ defmodule Cadet.Assessments do {:ok, nil} end) - transaction_result = Repo.transaction() - - case transaction_result do - {:ok, _result} -> - Logger.info("Successfully unsubmitting submission #{submission_id}") - - case submission.student_id do - # Team submission, handle notifications for team members - nil -> - Logger.info("Handling unsubmit notifications for team submission #{submission.id}") - team = Repo.get(Team, submission.team_id) - - query = - from(t in Team, - join: tm in TeamMember, - on: t.id == tm.team_id, - join: cr in CourseRegistration, - on: tm.student_id == cr.id, - where: t.id == ^team.id, - select: cr.id - ) - - team_members = Repo.all(query) - - Enum.each(team_members, fn tm_id -> - Logger.info("Sending unsubmit notification to team member #{tm_id}") - - Notifications.handle_unsubmit_notifications( - submission.assessment.id, - Repo.get(CourseRegistration, tm_id) - ) - end) - - student_id -> - Logger.info( - "Handling unsubmit notifications for individual submission #{submission.id}" + transaction_result = Repo.transaction(multi) + + case transaction_result do + {:ok, _result} -> + Logger.info("Successfully unsubmitting submission #{submission_id}") + + case submission.student_id do + # Team submission, handle notifications for team members + nil -> + Logger.info("Handling unsubmit notifications for team submission #{submission.id}") + team = Repo.get(Team, submission.team_id) + + query = + from(t in Team, + join: tm in TeamMember, + on: t.id == tm.team_id, + join: cr in CourseRegistration, + on: tm.student_id == cr.id, + where: t.id == ^team.id, + select: cr.id ) + team_members = Repo.all(query) + + Enum.each(team_members, fn tm_id -> + Logger.info("Sending unsubmit notification to team member #{tm_id}") + Notifications.handle_unsubmit_notifications( submission.assessment.id, - Repo.get(CourseRegistration, student_id) + Repo.get(CourseRegistration, tm_id) ) - end - - Logger.info("Removing grading notifications for submission #{submission.id}") - - # Remove grading notifications for submissions - Notification - |> where(submission_id: ^submission_id, type: :submitted) - |> select([n], n.id) - |> Repo.all() - |> Notifications.acknowledge(cr) - - {:ok, nil} - - {:error, _failed_operation, failed_value, _changes_so_far} -> - Logger.error("Failed to unsubmit submission #{submission_id}: #{inspect(failed_value)}") - {:error, {:internal_server_error, "Failed to unsubmit submission"}} - end - else - {:submission_found?, false} -> - Logger.error("Submission #{submission_id} not found") - {:error, {:not_found, "Submission not found"}} - - {:is_open?, false} -> - Logger.error("Assessment for submission #{submission_id} is not open") - {:error, {:forbidden, "Assessment not open"}} + end) - {:status, :attempting} -> - Logger.error("Submission #{submission_id} is still attempting") - {:error, {:bad_request, "Some questions have not been attempted"}} + student_id -> + Logger.info( + "Handling unsubmit notifications for individual submission #{submission.id}" + ) - {:status, :attempted} -> - Logger.error("Submission #{submission_id} has already been attempted") - {:error, {:bad_request, "Assessment has not been submitted"}} + Notifications.handle_unsubmit_notifications( + submission.assessment.id, + Repo.get(CourseRegistration, student_id) + ) + end - {:allowed_to_unsubmit?, false} -> - Logger.error( - "User #{course_reg_id} is not allowed to unsubmit submission #{submission_id}" - ) + Logger.info("Removing grading notifications for submission #{submission.id}") - {:error, {:forbidden, "Only Avenger of student or Admin is permitted to unsubmit"}} + # Remove grading notifications for submissions + Notification + |> where(submission_id: ^submission_id, type: :submitted) + |> select([n], n.id) + |> Repo.all() + |> Notifications.acknowledge(cr) - {:is_grading_published?, true} -> - Logger.error("Grading for submission #{submission_id} has already been published") - {:error, {:forbidden, "Grading has not been unpublished"}} + {:ok, nil} - _ -> - Logger.error("An unknown error occurred while unsubmitting submission #{submission_id}") - {:error, {:internal_server_error, "Please try again later."}} + {:error, _failed_operation, failed_value, _changes_so_far} -> + Logger.error("Failed to unsubmit submission #{submission_id}: #{inspect(failed_value)}") + {:error, {:internal_server_error, "Failed to unsubmit submission"}} end end @@ -1676,7 +1687,7 @@ defmodule Cadet.Assessments do {:status, :submitted} <- {:status, submission.status}, {:is_manually_graded?, true} <- {:is_manually_graded?, submission.assessment.config.is_manually_graded}, - {:fully_graded?, true} <- {:fully_graded?, is_fully_graded?(submission_id)}, + {:fully_graded?, true} <- {:fully_graded?, fully_graded?(submission_id)}, {:allowed_to_publish?, true} <- {:allowed_to_publish?, role == :admin or bypass or @@ -3177,7 +3188,7 @@ defmodule Cadet.Assessments do end end - defp is_fully_graded?(submission_id) do + defp fully_graded?(submission_id) do submission = Submission |> Repo.get_by(id: submission_id) @@ -3198,7 +3209,7 @@ defmodule Cadet.Assessments do question_count == graded_count end - def is_fully_autograded?(submission_id) do + def fully_autograded?(submission_id) do submission = Submission |> Repo.get_by(id: submission_id) @@ -3264,7 +3275,7 @@ defmodule Cadet.Assessments do {:ok, _} <- Repo.update(changeset) do update_xp_bonus(submission) - if is_grading_auto_published and is_fully_graded?(submission_id) do + if is_grading_auto_published and fully_graded?(submission_id) do publish_grading(submission_id, cr) end @@ -3379,8 +3390,8 @@ defmodule Cadet.Assessments do end # Checks if an assessment is open and published. - @spec is_open?(Assessment.t()) :: boolean() - def is_open?(%Assessment{open_at: open_at, close_at: close_at, is_published: is_published}) do + @spec open?(Assessment.t()) :: boolean() + def open?(%Assessment{open_at: open_at, close_at: close_at, is_published: is_published}) do Timex.between?(Timex.now(), open_at, close_at, inclusive: :start) and is_published end diff --git a/lib/cadet/jobs/autograder/grading_job.ex b/lib/cadet/jobs/autograder/grading_job.ex index e15f33480..4d15731ea 100644 --- a/lib/cadet/jobs/autograder/grading_job.ex +++ b/lib/cadet/jobs/autograder/grading_job.ex @@ -323,7 +323,7 @@ defmodule Cadet.Autograder.GradingJob do is_grading_auto_published = assessment_config.is_grading_auto_published is_manually_graded = assessment_config.is_manually_graded - if Assessments.is_fully_autograded?(submission_id) and is_grading_auto_published and + if Assessments.fully_autograded?(submission_id) and is_grading_auto_published and not is_manually_graded do Assessments.publish_grading(submission_id) end diff --git a/lib/cadet/jobs/autograder/result_store_worker.ex b/lib/cadet/jobs/autograder/result_store_worker.ex index 500004d50..0e4649cbf 100644 --- a/lib/cadet/jobs/autograder/result_store_worker.ex +++ b/lib/cadet/jobs/autograder/result_store_worker.ex @@ -96,7 +96,7 @@ defmodule Cadet.Autograder.ResultStoreWorker do is_grading_auto_published = assessment_config.is_grading_auto_published is_manually_graded = assessment_config.is_manually_graded - if Assessments.is_fully_autograded?(submission_id) and is_grading_auto_published and + if Assessments.fully_autograded?(submission_id) and is_grading_auto_published and not is_manually_graded do Assessments.publish_grading(submission_id) end diff --git a/lib/cadet/workers/NotificationWorker.ex b/lib/cadet/workers/NotificationWorker.ex index ad402a8fb..c240e1698 100644 --- a/lib/cadet/workers/NotificationWorker.ex +++ b/lib/cadet/workers/NotificationWorker.ex @@ -6,11 +6,11 @@ defmodule Cadet.Workers.NotificationWorker do alias Cadet.{Email, Notifications, Mailer} alias Cadet.Repo - defp is_system_enabled(notification_type_id) do + defp system_enabled?(notification_type_id) do Notifications.get_notification_type!(notification_type_id).is_enabled end - defp is_course_enabled(notification_type_id, course_id, assessment_config_id) do + defp course_enabled?(notification_type_id, course_id, assessment_config_id) do notification_config = Notifications.get_notification_config!( notification_type_id, @@ -25,7 +25,7 @@ defmodule Cadet.Workers.NotificationWorker do end end - defp is_user_enabled(notification_type_id, course_reg_id) do + defp user_enabled?(notification_type_id, course_reg_id) do pref = Notifications.get_notification_preference(notification_type_id, course_reg_id) if is_nil(pref) do @@ -37,7 +37,7 @@ defmodule Cadet.Workers.NotificationWorker do # Returns true if user preference matches the job's time option. # If user has made no preference, the default time option is used instead - def is_user_time_option_matched( + def user_time_option_matched?( notification_type_id, assessment_config_id, course_reg_id, @@ -65,9 +65,9 @@ defmodule Cadet.Workers.NotificationWorker do ntype = Cadet.Notifications.get_notification_type_by_name!("AVENGER BACKLOG") notification_type_id = ntype.id - if is_system_enabled(notification_type_id) do + if system_enabled?(notification_type_id) do for course_id <- Cadet.Courses.get_all_course_ids() do - if is_course_enabled(notification_type_id, course_id, nil) do + if course_enabled?(notification_type_id, course_id, nil) do avengers_crs = Cadet.Accounts.CourseRegistrations.get_staffs(course_id) for avenger_cr <- avengers_crs do @@ -118,7 +118,7 @@ defmodule Cadet.Workers.NotificationWorker do notification_type = Cadet.Notifications.get_notification_type_by_name!("ASSESSMENT SUBMISSION") - if is_system_enabled(notification_type.id) do + if system_enabled?(notification_type.id) do submission = Cadet.Assessments.get_submission_by_id(submission_id) course_id = submission.assessment.course_id student_id = submission.student_id @@ -129,10 +129,10 @@ defmodule Cadet.Workers.NotificationWorker do avenger = avenger_cr.user cond do - !is_course_enabled(notification_type.id, course_id, assessment_config_id) -> + !course_enabled?(notification_type.id, course_id, assessment_config_id) -> IO.puts("[ASSESSMENT_SUBMISSION] course-level disabled") - !is_user_enabled(notification_type.id, avenger_cr.id) -> + !user_enabled?(notification_type.id, avenger_cr.id) -> IO.puts("[ASSESSMENT_SUBMISSION] user-level disabled") true -> diff --git a/lib/cadet_web/admin_controllers/admin_assessments_controller.ex b/lib/cadet_web/admin_controllers/admin_assessments_controller.ex index 862ad7444..b2223e8da 100644 --- a/lib/cadet_web/admin_controllers/admin_assessments_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_assessments_controller.ex @@ -63,7 +63,7 @@ defmodule CadetWeb.AdminAssessmentsController do end def delete(conn, %{"course_id" => course_id, "assessmentid" => assessment_id}) do - with {:same_course, true} <- {:same_course, is_same_course(course_id, assessment_id)}, + with {:same_course, true} <- {:same_course, same_course?(course_id, assessment_id)}, {:ok, _} <- Assessments.delete_assessment(assessment_id) do text(conn, "OK") else @@ -183,7 +183,7 @@ defmodule CadetWeb.AdminAssessmentsController do end end - defp is_same_course(course_id, assessment_id) do + defp same_course?(course_id, assessment_id) do Assessment |> where(id: ^assessment_id) |> where(course_id: ^course_id) diff --git a/lib/cadet_web/controllers/answer_controller.ex b/lib/cadet_web/controllers/answer_controller.ex index c4c99f03f..7c7a85354 100644 --- a/lib/cadet_web/controllers/answer_controller.ex +++ b/lib/cadet_web/controllers/answer_controller.ex @@ -17,7 +17,7 @@ defmodule CadetWeb.AnswerController do with {:question, question} when not is_nil(question) <- {:question, Assessments.get_question(question_id)}, {:is_open?, true} <- - {:is_open?, can_bypass? or Assessments.is_open?(question.assessment)}, + {:is_open?, can_bypass? or Assessments.open?(question.assessment)}, {:ok, _nil} <- Assessments.answer_question(question, course_reg, answer, can_bypass?) do text(conn, "OK") else @@ -53,7 +53,7 @@ defmodule CadetWeb.AnswerController do with {:question, question} when not is_nil(question) <- {:question, Assessments.get_question(question_id)}, {:is_open?, true} <- - {:is_open?, can_bypass? or Assessments.is_open?(question.assessment)}, + {:is_open?, can_bypass? or Assessments.open?(question.assessment)}, {:ok, last_modified} <- Assessments.has_last_modified_answer?( question, diff --git a/lib/cadet_web/controllers/assessments_controller.ex b/lib/cadet_web/controllers/assessments_controller.ex index 76e746dfe..b5f613814 100644 --- a/lib/cadet_web/controllers/assessments_controller.ex +++ b/lib/cadet_web/controllers/assessments_controller.ex @@ -22,7 +22,7 @@ defmodule CadetWeb.AssessmentsController do {:submission, Assessments.get_submission(assessment_id, cr)}, {:is_open?, true} <- {:is_open?, - cr.role in @bypass_closed_roles or Assessments.is_open?(submission.assessment)}, + cr.role in @bypass_closed_roles or Assessments.open?(submission.assessment)}, {:ok, _nil} <- Assessments.finalise_submission(submission) do Logger.info("Successfully submitted assessment #{assessment_id} for user #{cr.id}.") diff --git a/test/cadet/assessments/assessments_test.exs b/test/cadet/assessments/assessments_test.exs index 202275047..5594e8ff2 100644 --- a/test/cadet/assessments/assessments_test.exs +++ b/test/cadet/assessments/assessments_test.exs @@ -3066,7 +3066,7 @@ defmodule Cadet.AssessmentsTest do end end - describe "is_fully_autograded? function" do + describe "fully_autograded? function" do setup do assessment = insert(:assessment) student = insert(:course_registration, role: :student) @@ -3087,7 +3087,7 @@ defmodule Cadet.AssessmentsTest do insert(:answer, submission: submission, question: question, autograding_status: :success) insert(:answer, submission: submission, question: question2, autograding_status: :success) - assert Assessments.is_fully_autograded?(submission.id) == true + assert Assessments.fully_autograded?(submission.id) == true end test "returns false when not all answers are autograded successfully", %{ @@ -3098,7 +3098,7 @@ defmodule Cadet.AssessmentsTest do insert(:answer, submission: submission, question: question, autograding_status: :success) insert(:answer, submission: submission, question: question2, autograding_status: :failed) - assert Assessments.is_fully_autograded?(submission.id) == false + assert Assessments.fully_autograded?(submission.id) == false end test "returns false when not all answers are autograded successfully 2", %{ @@ -3109,7 +3109,7 @@ defmodule Cadet.AssessmentsTest do insert(:answer, submission: submission, question: question, autograding_status: :success) insert(:answer, submission: submission, question: question2, autograding_status: :none) - assert Assessments.is_fully_autograded?(submission.id) == false + assert Assessments.fully_autograded?(submission.id) == false end end From 73f58691c6d713977b3f6beeb8f8888895bc9e22 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Thu, 26 Mar 2026 16:59:32 +0800 Subject: [PATCH 45/68] pass testcases for llmcost calculations --- lib/cadet/assessments/assessments.ex | 2 +- test/test_helper.exs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 74ecd903c..f6b0a0449 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -1472,7 +1472,7 @@ defmodule Cadet.Assessments do perform_unsubmit_transaction(submission, submission_id, cr) {:error, reason} -> - reason + {:error, reason} end end diff --git a/test/test_helper.exs b/test/test_helper.exs index 6004b70aa..62706dbbc 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -13,13 +13,13 @@ case Cadet.Repo.start_link() do {:error, {:already_started, _pid}} -> :ok end -case Ecto.Adapters.Postgres.storage_down(Cadet.Repo) do +case Ecto.Adapters.Postgres.storage_down(Cadet.Repo.config()) do :ok -> :ok {:error, :already_down} -> :ok {:error, _} -> :ok end -case Ecto.Adapters.Postgres.storage_up(Cadet.Repo) do +case Ecto.Adapters.Postgres.storage_up(Cadet.Repo.config()) do :ok -> :ok {:error, :already_up} -> :ok {:error, _} -> :ok From 058b83629a315912b9f0208d4a2e21e86629bf2c Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Thu, 26 Mar 2026 17:06:42 +0800 Subject: [PATCH 46/68] pass testcases for llmcost calculations --- lib/cadet/assessments/assessments.ex | 73 ++++++++++++++-------------- test/test_helper.exs | 2 +- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index f6b0a0449..fd81d70de 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -1483,47 +1483,48 @@ defmodule Cadet.Assessments do |> preload([_, a], assessment: a) |> Repo.get(submission_id) - # allows staff to unsubmit own assessment - bypass = role in @bypass_closed_roles and submission.student_id == cr.id + with {:submission_found?, true} <- {:submission_found?, is_map(submission)} do + # allows staff to unsubmit own assessment + bypass = role in @bypass_closed_roles and submission.student_id == cr.id + + with {:is_open?, true} <- {:is_open?, bypass or open?(submission.assessment)}, + {:status, :submitted} <- {:status, submission.status}, + {:allowed_to_unsubmit?, true} <- + {:allowed_to_unsubmit?, + role == :admin or bypass or is_nil(submission.student_id) or + Cadet.Accounts.Query.avenger_of?(cr, submission.student_id)}, + {:is_grading_published?, false} <- + {:is_grading_published?, submission.is_grading_published} do + {:ok, submission} + else + {:is_open?, false} -> + Logger.error("Assessment for submission #{submission_id} is not open") + {:error, {:forbidden, "Assessment not open"}} - with {:submission_found?, true} <- {:submission_found?, is_map(submission)}, - {:is_open?, true} <- {:is_open?, bypass or open?(submission.assessment)}, - {:status, :submitted} <- {:status, submission.status}, - {:allowed_to_unsubmit?, true} <- - {:allowed_to_unsubmit?, - role == :admin or bypass or is_nil(submission.student_id) or - Cadet.Accounts.Query.avenger_of?(cr, submission.student_id)}, - {:is_grading_published?, false} <- - {:is_grading_published?, submission.is_grading_published} do - {:ok, submission} - else - {:submission_found?, false} -> - Logger.error("Submission #{submission_id} not found") - {:error, {:not_found, "Submission not found"}} + {:status, :attempting} -> + Logger.error("Submission #{submission_id} is still attempting") + {:error, {:bad_request, "Some questions have not been attempted"}} - {:is_open?, false} -> - Logger.error("Assessment for submission #{submission_id} is not open") - {:error, {:forbidden, "Assessment not open"}} + {:status, :attempted} -> + Logger.error("Submission #{submission_id} has already been attempted") + {:error, {:bad_request, "Assessment has not been submitted"}} - {:status, :attempting} -> - Logger.error("Submission #{submission_id} is still attempting") - {:error, {:bad_request, "Some questions have not been attempted"}} + {:allowed_to_unsubmit?, false} -> + Logger.error("User #{cr.id} is not allowed to unsubmit submission #{submission_id}") + {:error, {:forbidden, "Only Avenger of student or Admin is permitted to unsubmit"}} - {:status, :attempted} -> - Logger.error("Submission #{submission_id} has already been attempted") - {:error, {:bad_request, "Assessment has not been submitted"}} - - {:allowed_to_unsubmit?, false} -> - Logger.error("User #{cr.id} is not allowed to unsubmit submission #{submission_id}") - {:error, {:forbidden, "Only Avenger of student or Admin is permitted to unsubmit"}} - - {:is_grading_published?, true} -> - Logger.error("Grading for submission #{submission_id} has already been published") - {:error, {:forbidden, "Grading has not been unpublished"}} + {:is_grading_published?, true} -> + Logger.error("Grading for submission #{submission_id} has already been published") + {:error, {:forbidden, "Grading has not been unpublished"}} - _ -> - Logger.error("An unknown error occurred while unsubmitting submission #{submission_id}") - {:error, {:internal_server_error, "Please try again later."}} + _ -> + Logger.error("An unknown error occurred while unsubmitting submission #{submission_id}") + {:error, {:internal_server_error, "Please try again later."}} + end + else + {:submission_found?, false} -> + Logger.error("Submission #{submission_id} not found") + {:error, {:not_found, "Submission not found"}} end end diff --git a/test/test_helper.exs b/test/test_helper.exs index 62706dbbc..c6766c617 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -26,6 +26,6 @@ case Ecto.Adapters.Postgres.storage_up(Cadet.Repo.config()) do end # Run all pending migrations -:ok = Ecto.Migrator.run(Cadet.Repo, :up, all: true) +Ecto.Migrator.run(Cadet.Repo, :up, all: true) Ecto.Adapters.SQL.Sandbox.mode(Cadet.Repo, :manual) From e766b8238f454917f01c1950886b051e0887e72f Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Thu, 26 Mar 2026 17:15:21 +0800 Subject: [PATCH 47/68] pass testcases for llmcost calculations --- lib/cadet/assessments/assessments.ex | 74 +++++++++++++++------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index fd81d70de..a44450f13 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -1483,46 +1483,50 @@ defmodule Cadet.Assessments do |> preload([_, a], assessment: a) |> Repo.get(submission_id) - with {:submission_found?, true} <- {:submission_found?, is_map(submission)} do - # allows staff to unsubmit own assessment - bypass = role in @bypass_closed_roles and submission.student_id == cr.id - - with {:is_open?, true} <- {:is_open?, bypass or open?(submission.assessment)}, - {:status, :submitted} <- {:status, submission.status}, - {:allowed_to_unsubmit?, true} <- - {:allowed_to_unsubmit?, - role == :admin or bypass or is_nil(submission.student_id) or - Cadet.Accounts.Query.avenger_of?(cr, submission.student_id)}, - {:is_grading_published?, false} <- - {:is_grading_published?, submission.is_grading_published} do - {:ok, submission} - else - {:is_open?, false} -> - Logger.error("Assessment for submission #{submission_id} is not open") - {:error, {:forbidden, "Assessment not open"}} + case is_map(submission) do + true -> + # allows staff to unsubmit own assessment + bypass = role in @bypass_closed_roles and submission.student_id == cr.id + + with {:is_open?, true} <- {:is_open?, bypass or open?(submission.assessment)}, + {:status, :submitted} <- {:status, submission.status}, + {:allowed_to_unsubmit?, true} <- + {:allowed_to_unsubmit?, + role == :admin or bypass or is_nil(submission.student_id) or + Cadet.Accounts.Query.avenger_of?(cr, submission.student_id)}, + {:is_grading_published?, false} <- + {:is_grading_published?, submission.is_grading_published} do + {:ok, submission} + else + {:is_open?, false} -> + Logger.error("Assessment for submission #{submission_id} is not open") + {:error, {:forbidden, "Assessment not open"}} - {:status, :attempting} -> - Logger.error("Submission #{submission_id} is still attempting") - {:error, {:bad_request, "Some questions have not been attempted"}} + {:status, :attempting} -> + Logger.error("Submission #{submission_id} is still attempting") + {:error, {:bad_request, "Some questions have not been attempted"}} - {:status, :attempted} -> - Logger.error("Submission #{submission_id} has already been attempted") - {:error, {:bad_request, "Assessment has not been submitted"}} + {:status, :attempted} -> + Logger.error("Submission #{submission_id} has already been attempted") + {:error, {:bad_request, "Assessment has not been submitted"}} - {:allowed_to_unsubmit?, false} -> - Logger.error("User #{cr.id} is not allowed to unsubmit submission #{submission_id}") - {:error, {:forbidden, "Only Avenger of student or Admin is permitted to unsubmit"}} + {:allowed_to_unsubmit?, false} -> + Logger.error("User #{cr.id} is not allowed to unsubmit submission #{submission_id}") + {:error, {:forbidden, "Only Avenger of student or Admin is permitted to unsubmit"}} - {:is_grading_published?, true} -> - Logger.error("Grading for submission #{submission_id} has already been published") - {:error, {:forbidden, "Grading has not been unpublished"}} + {:is_grading_published?, true} -> + Logger.error("Grading for submission #{submission_id} has already been published") + {:error, {:forbidden, "Grading has not been unpublished"}} - _ -> - Logger.error("An unknown error occurred while unsubmitting submission #{submission_id}") - {:error, {:internal_server_error, "Please try again later."}} - end - else - {:submission_found?, false} -> + _ -> + Logger.error( + "An unknown error occurred while unsubmitting submission #{submission_id}" + ) + + {:error, {:internal_server_error, "Please try again later."}} + end + + false -> Logger.error("Submission #{submission_id} not found") {:error, {:not_found, "Submission not found"}} end From bd8a82bf0c0051c1eead5759ddf6c2769363f272 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Thu, 26 Mar 2026 17:24:07 +0800 Subject: [PATCH 48/68] pass testcases for llmcost calculations --- lib/cadet_web/views/assessments_view.ex | 32 ++++++++++++++++++------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/lib/cadet_web/views/assessments_view.ex b/lib/cadet_web/views/assessments_view.ex index 215c46c86..547359215 100644 --- a/lib/cadet_web/views/assessments_view.ex +++ b/lib/cadet_web/views/assessments_view.ex @@ -9,7 +9,7 @@ defmodule CadetWeb.AssessmentsView do end def render("overview.json", %{assessment: assessment}) do - transform_map_for_view(assessment, %{ + base_map = %{ id: :id, courseId: :course_id, title: :title, @@ -36,14 +36,28 @@ defmodule CadetWeb.AssessmentsView do hasTokenCounter: :has_token_counter, isVotingPublished: :is_voting_published, hoursBeforeEarlyXpDecay: & &1.config.hours_before_early_xp_decay, - isLlmGraded: &(&1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""]), - llmInputCost: :llm_input_cost, - llmOutputCost: :llm_output_cost, - llmTotalInputTokens: :llm_total_input_tokens, - llmTotalOutputTokens: :llm_total_output_tokens, - llmTotalCachedTokens: :llm_total_cached_tokens, - llmTotalCost: :llm_total_cost - }) + isLlmGraded: &(&1.has_llm_questions || &1.llm_assessment_prompt not in [nil, ""]) + } + + # Only include LLM cost fields if the assessment is LLM graded + is_llm_graded = + assessment.has_llm_questions || assessment.llm_assessment_prompt not in [nil, ""] + + final_map = + if is_llm_graded do + Map.merge(base_map, %{ + llmInputCost: :llm_input_cost, + llmOutputCost: :llm_output_cost, + llmTotalInputTokens: :llm_total_input_tokens, + llmTotalOutputTokens: :llm_total_output_tokens, + llmTotalCachedTokens: :llm_total_cached_tokens, + llmTotalCost: :llm_total_cost + }) + else + base_map + end + + transform_map_for_view(assessment, final_map) end def render("show.json", %{assessment: assessment}) do From 70c50ffb6bfa93eebd53ef7ba3ae62e4156c75a0 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Fri, 27 Mar 2026 09:42:01 +0800 Subject: [PATCH 49/68] pass testcases for llmcost calculations --- lib/cadet/assessments/assessments.ex | 4 +++- ...20260320075234_add_detailed_llm_costs_to_assessments.exs | 6 +++--- .../20260320094651_repair_llm_columns_on_assessments.exs | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index a44450f13..972854d70 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3672,7 +3672,9 @@ defmodule Cadet.Assessments do input_rate = get_valid_rate(assessment.llm_input_cost, "3.20") output_rate = get_valid_rate(assessment.llm_output_cost, "12.80") - new_cost = calculate_token_cost(prompt, completion, input_rate, output_rate) + new_cost = + calculate_token_cost(prompt, completion, input_rate, output_rate) + |> Decimal.round(6, :half_up) # Atomic database-level updates to prevent race conditions # All increments happen in a single transaction at the database level diff --git a/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs b/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs index 9b25c6316..854043849 100644 --- a/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs +++ b/priv/repo/migrations/20260320075234_add_detailed_llm_costs_to_assessments.exs @@ -3,12 +3,12 @@ defmodule Cadet.Repo.Migrations.RepairLlmColumnsOnAssessments do def up do alter table(:assessments) do - add_if_not_exists(:llm_input_cost, :decimal, precision: 10, scale: 4, default: 3.20) - add_if_not_exists(:llm_output_cost, :decimal, precision: 10, scale: 4, default: 12.80) + add_if_not_exists(:llm_input_cost, :decimal, precision: 10, scale: 6, default: 3.20) + add_if_not_exists(:llm_output_cost, :decimal, precision: 10, scale: 6, default: 12.80) add_if_not_exists(:llm_total_input_tokens, :integer, default: 0) add_if_not_exists(:llm_total_output_tokens, :integer, default: 0) add_if_not_exists(:llm_total_cached_tokens, :integer, default: 0) - add_if_not_exists(:llm_total_cost, :decimal, precision: 10, scale: 4, default: 0.0) + add_if_not_exists(:llm_total_cost, :decimal, precision: 10, scale: 6, default: 0.0) end end diff --git a/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs b/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs index 9b25c6316..854043849 100644 --- a/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs +++ b/priv/repo/migrations/20260320094651_repair_llm_columns_on_assessments.exs @@ -3,12 +3,12 @@ defmodule Cadet.Repo.Migrations.RepairLlmColumnsOnAssessments do def up do alter table(:assessments) do - add_if_not_exists(:llm_input_cost, :decimal, precision: 10, scale: 4, default: 3.20) - add_if_not_exists(:llm_output_cost, :decimal, precision: 10, scale: 4, default: 12.80) + add_if_not_exists(:llm_input_cost, :decimal, precision: 10, scale: 6, default: 3.20) + add_if_not_exists(:llm_output_cost, :decimal, precision: 10, scale: 6, default: 12.80) add_if_not_exists(:llm_total_input_tokens, :integer, default: 0) add_if_not_exists(:llm_total_output_tokens, :integer, default: 0) add_if_not_exists(:llm_total_cached_tokens, :integer, default: 0) - add_if_not_exists(:llm_total_cost, :decimal, precision: 10, scale: 4, default: 0.0) + add_if_not_exists(:llm_total_cost, :decimal, precision: 10, scale: 6, default: 0.0) end end From 296cd17a274ff87766bcd511f62bec650b56288d Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Fri, 27 Mar 2026 09:48:44 +0800 Subject: [PATCH 50/68] pass testcases for llmcost calculations --- lib/cadet/assessments/assessments.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 972854d70..fe71c0fca 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3672,9 +3672,8 @@ defmodule Cadet.Assessments do input_rate = get_valid_rate(assessment.llm_input_cost, "3.20") output_rate = get_valid_rate(assessment.llm_output_cost, "12.80") - new_cost = - calculate_token_cost(prompt, completion, input_rate, output_rate) - |> Decimal.round(6, :half_up) + raw_cost = calculate_token_cost(prompt, completion, input_rate, output_rate) + new_cost = Decimal.round(raw_cost, 6, :half_up) # Atomic database-level updates to prevent race conditions # All increments happen in a single transaction at the database level From 4da9d4f97270b78eb41e35d22d201c3123b5f805 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Fri, 27 Mar 2026 10:04:15 +0800 Subject: [PATCH 51/68] pass testcases for llmcost calculations --- lib/cadet/assessments/assessments.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index fe71c0fca..ba8087b0e 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -1460,6 +1460,7 @@ defmodule Cadet.Assessments do end @dialyzer {:nowarn_function, unsubmit_submission: 2} + @dialyzer {:nowarn_function, perform_unsubmit_transaction: 3} def unsubmit_submission( submission_id, cr = %CourseRegistration{id: course_reg_id, role: role} From bcf078b05e9d87b776a31802cf299ebb86af2c71 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 12:22:35 +0800 Subject: [PATCH 52/68] Added a new tab for coursewide summary for LLM stats --- lib/cadet/assessments/query.ex | 20 +++ lib/cadet/courses/course.ex | 3 + lib/cadet/courses/courses.ex | 8 +- lib/cadet/llm_stats.ex | 168 ++++++++++++++++++ .../admin_llm_stats_controller.ex | 5 + lib/cadet_web/router.ex | 1 + lib/cadet_web/views/courses_view.ex | 1 + lib/cadet_web/views/user_view.ex | 7 +- test/cadet/assessments/query_test.exs | 42 +++++ test/cadet/llm_stats_test.exs | 165 +++++++++++++++++ .../admin_llm_stats_controller_test.exs | 78 ++++++++ .../controllers/courses_controller_test.exs | 39 ++++ .../controllers/user_controller_test.exs | 24 +++ 13 files changed, 559 insertions(+), 2 deletions(-) diff --git a/lib/cadet/assessments/query.ex b/lib/cadet/assessments/query.ex index b93c20d51..b5cd7639c 100644 --- a/lib/cadet/assessments/query.ex +++ b/lib/cadet/assessments/query.ex @@ -59,4 +59,24 @@ defmodule Cadet.Assessments.Query do ) }) end + + @doc """ + Checks if a course has any assessments with LLM content. + Returns true if any assessment has questions with llm_prompt or llm_assessment_prompt. + """ + @spec course_has_llm_content?(integer()) :: boolean() + def course_has_llm_content?(course_id) when is_ecto_id(course_id) do + Assessment + |> where(course_id: ^course_id) + |> join(:left, [a], q in subquery(assessments_aggregates()), on: a.id == q.assessment_id) + |> select([a, q], %{ + has_llm_questions: q.has_llm_questions, + llm_assessment_prompt: a.llm_assessment_prompt + }) + |> Repo.all() + |> Enum.any?(fn assessment -> + assessment.has_llm_questions == true or + assessment.llm_assessment_prompt not in [nil, ""] + end) + end end diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index b8a113be1..7cb16b2d6 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -54,6 +54,9 @@ defmodule Cadet.Courses.Course do # for now, only settable from database field(:assets_prefix, :string, default: nil) + # Virtual field computed at runtime based on assessments in course + field(:has_llm_content, :boolean, virtual: true, default: false) + has_many(:assessment_config, AssessmentConfig) timestamps() diff --git a/lib/cadet/courses/courses.ex b/lib/cadet/courses/courses.ex index 965110673..fe483883b 100644 --- a/lib/cadet/courses/courses.ex +++ b/lib/cadet/courses/courses.ex @@ -77,8 +77,14 @@ defmodule Cadet.Courses do |> Enum.sort(&(&1.order < &2.order)) |> Enum.map(& &1.type) + has_llm_content = Assessments.Query.course_has_llm_content?(course_id) + Logger.info("Successfully retrieved course configuration for course #{course_id}") - {:ok, Map.put_new(course, :assessment_configs, assessment_configs)} + + {:ok, + course + |> Map.put(:assessment_configs, assessment_configs) + |> Map.put(:has_llm_content, has_llm_content)} end end diff --git a/lib/cadet/llm_stats.ex b/lib/cadet/llm_stats.ex index b0b6eec31..97fc9b951 100644 --- a/lib/cadet/llm_stats.ex +++ b/lib/cadet/llm_stats.ex @@ -106,6 +106,174 @@ defmodule Cadet.LLMStats do } end + def get_course_statistics(course_id) do + assessments = + from(a in Cadet.Assessments.Assessment, + where: a.course_id == ^course_id and a.is_published == true, + where: + fragment("? IS NOT NULL AND ? != ''", a.llm_assessment_prompt, a.llm_assessment_prompt) or + fragment( + "EXISTS (SELECT 1 FROM questions q WHERE q.assessment_id = ? AND q.question ->> 'llm_prompt' IS NOT NULL AND q.question ->> 'llm_prompt' != '')", + a.id + ), + join: c in assoc(a, :config), + select: %{ + assessment_id: a.id, + title: a.title, + category: c.type, + llm_total_input_tokens: coalesce(a.llm_total_input_tokens, 0), + llm_total_output_tokens: coalesce(a.llm_total_output_tokens, 0), + llm_total_cost: coalesce(a.llm_total_cost, 0) + } + ) + |> Repo.all() + + assessments_with_stats = + Enum.map(assessments, fn assessment -> + total_uses = + Repo.one( + from(l in LLMUsageLog, + where: l.course_id == ^course_id and l.assessment_id == ^assessment.assessment_id, + select: count(l.id) + ) + ) || 0 + + avg_rating = + Repo.one( + from(f in LLMFeedback, + where: + f.course_id == ^course_id and f.assessment_id == ^assessment.assessment_id and + not is_nil(f.rating), + select: avg(f.rating) + ) + ) + + avg_rating = + if is_nil(avg_rating) do + nil + else + avg_rating |> Decimal.to_float() |> Float.round(2) + end + + questions = + Repo.all( + from(q in Cadet.Assessments.Question, + where: q.assessment_id == ^assessment.assessment_id, + where: + fragment( + "? ->> 'llm_prompt' IS NOT NULL AND ? ->> 'llm_prompt' != ''", + q.question, + q.question + ), + order_by: [asc: q.display_order], + select: %{ + question_id: q.id, + display_order: q.display_order + } + ) + ) + + question_stats = + Enum.map(questions, fn question -> + question_uses = + Repo.one( + from(l in LLMUsageLog, + where: + l.course_id == ^course_id and l.assessment_id == ^assessment.assessment_id and + l.question_id == ^question.question_id, + select: count(l.id) + ) + ) || 0 + + question_rating = + Repo.one( + from(f in LLMFeedback, + where: + f.course_id == ^course_id and f.assessment_id == ^assessment.assessment_id and + f.question_id == ^question.question_id and not is_nil(f.rating), + select: avg(f.rating) + ) + ) + + question_rating = + if is_nil(question_rating) do + nil + else + question_rating |> Decimal.to_float() |> Float.round(2) + end + + question_input_tokens = + if total_uses > 0 do + round(assessment.llm_total_input_tokens * question_uses / total_uses) + else + 0 + end + + question_output_tokens = + if total_uses > 0 do + round(assessment.llm_total_output_tokens * question_uses / total_uses) + else + 0 + end + + question_cost = + if total_uses > 0 do + Decimal.mult( + assessment.llm_total_cost, + Decimal.div(Decimal.new(question_uses), Decimal.new(total_uses)) + ) + |> Decimal.round(6, :half_up) + else + Decimal.new("0.0") + end + + %{ + question_id: question.question_id, + display_order: question.display_order, + total_uses: question_uses, + avg_rating: question_rating, + llm_total_input_tokens: question_input_tokens, + llm_total_output_tokens: question_output_tokens, + llm_total_cost: question_cost + } + end) + + %{ + assessment_id: assessment.assessment_id, + title: assessment.title, + category: assessment.category, + total_uses: total_uses, + avg_rating: avg_rating, + llm_total_input_tokens: assessment.llm_total_input_tokens, + llm_total_output_tokens: assessment.llm_total_output_tokens, + llm_total_cost: assessment.llm_total_cost, + questions: question_stats + } + end) + + course_total_input_tokens = + Enum.reduce(assessments_with_stats, 0, fn assessment, acc -> + acc + assessment.llm_total_input_tokens + end) + + course_total_output_tokens = + Enum.reduce(assessments_with_stats, 0, fn assessment, acc -> + acc + assessment.llm_total_output_tokens + end) + + course_total_cost = + Enum.reduce(assessments_with_stats, Decimal.new("0.0"), fn assessment, acc -> + Decimal.add(acc, assessment.llm_total_cost) + end) + + %{ + course_total_input_tokens: course_total_input_tokens, + course_total_output_tokens: course_total_output_tokens, + course_total_cost: course_total_cost, + assessments: assessments_with_stats + } + end + # ===================== # Question-level Statistics # ===================== diff --git a/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex b/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex index 3046d6c48..6dfb8c8a4 100644 --- a/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex @@ -12,6 +12,11 @@ defmodule CadetWeb.AdminLLMStatsController do GET /admin/llm-stats/:assessment_id Returns assessment-level LLM usage statistics with per-question breakdown. """ + def course_stats(conn, %{"course_id" => course_id}) do + stats = LLMStats.get_course_statistics(course_id) + json(conn, stats) + end + def assessment_stats(conn, %{"course_id" => course_id, "assessment_id" => assessment_id}) do stats = LLMStats.get_assessment_statistics(course_id, assessment_id) json(conn, stats) diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index cdbe0b871..9509cd6e7 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -239,6 +239,7 @@ defmodule CadetWeb.Router do ) # LLM Statistics & Feedback (per-assessment) + get("/llm-stats", AdminLLMStatsController, :course_stats) get("/llm-stats/:assessment_id", AdminLLMStatsController, :assessment_stats) get("/llm-stats/:assessment_id/feedback", AdminLLMStatsController, :get_feedback) post("/llm-stats/:assessment_id/feedback", AdminLLMStatsController, :submit_feedback) diff --git a/lib/cadet_web/views/courses_view.ex b/lib/cadet_web/views/courses_view.ex index a3a3f443e..75953336b 100644 --- a/lib/cadet_web/views/courses_view.ex +++ b/lib/cadet_web/views/courses_view.ex @@ -17,6 +17,7 @@ defmodule CadetWeb.CoursesView do enableSourcecast: :enable_sourcecast, enableStories: :enable_stories, enableLlmGrading: :enable_llm_grading, + hasLlmContent: :has_llm_content, llmModel: :llm_model, llmApiUrl: :llm_api_url, llmCourseLevelPrompt: :llm_course_level_prompt, diff --git a/lib/cadet_web/views/user_view.ex b/lib/cadet_web/views/user_view.ex index 4a497465b..ed7f5f3be 100644 --- a/lib/cadet_web/views/user_view.ex +++ b/lib/cadet_web/views/user_view.ex @@ -1,6 +1,7 @@ defmodule CadetWeb.UserView do use CadetWeb, :view + alias Cadet.Assessments.Query alias Cadet.Courses def render("index.json", %{ @@ -97,7 +98,9 @@ defmodule CadetWeb.UserView do nil _ -> - transform_map_for_view(latest.course, %{ + latest.course + |> Map.put(:has_llm_content, Query.course_has_llm_content?(latest.course.id)) + |> transform_map_for_view(%{ courseName: :course_name, courseShortName: :course_short_name, viewable: :viewable, @@ -109,6 +112,8 @@ defmodule CadetWeb.UserView do topContestLeaderboardDisplay: :top_contest_leaderboard_display, enableSourcecast: :enable_sourcecast, enableStories: :enable_stories, + enableLlmGrading: :enable_llm_grading, + hasLlmContent: :has_llm_content, sourceChapter: :source_chapter, sourceVariant: :source_variant, moduleHelpText: :module_help_text, diff --git a/test/cadet/assessments/query_test.exs b/test/cadet/assessments/query_test.exs index aed849a09..d1d085bdb 100644 --- a/test/cadet/assessments/query_test.exs +++ b/test/cadet/assessments/query_test.exs @@ -74,4 +74,46 @@ defmodule Cadet.Assessments.QueryTest do assert result.has_llm_questions == false end + + test "course_has_llm_content? returns false when course has no assessments" do + course = insert(:course) + + assert Query.course_has_llm_content?(course.id) == false + end + + test "course_has_llm_content? returns true when assessment has non-empty llm_assessment_prompt" do + course = insert(:course) + insert(:assessment, course: course, llm_assessment_prompt: "Use this grading rubric") + + assert Query.course_has_llm_content?(course.id) == true + end + + test "course_has_llm_content? returns true when any question has non-empty llm_prompt" do + course = insert(:course) + assessment = insert(:assessment, course: course, llm_assessment_prompt: nil) + + insert(:question, + assessment: assessment, + question: build(:programming_question_content, llm_prompt: "Provide AI feedback") + ) + + assert Query.course_has_llm_content?(course.id) == true + end + + test "course_has_llm_content? returns false when llm_assessment_prompt is empty and question llm_prompt values are nil or empty" do + course = insert(:course) + assessment = insert(:assessment, course: course, llm_assessment_prompt: "") + + insert(:question, + assessment: assessment, + question: build(:programming_question_content, llm_prompt: nil) + ) + + insert(:question, + assessment: assessment, + question: build(:programming_question_content, llm_prompt: "") + ) + + assert Query.course_has_llm_content?(course.id) == false + end end diff --git a/test/cadet/llm_stats_test.exs b/test/cadet/llm_stats_test.exs index c5a381533..c6a315f62 100644 --- a/test/cadet/llm_stats_test.exs +++ b/test/cadet/llm_stats_test.exs @@ -133,6 +133,171 @@ defmodule Cadet.LLMStatsTest do end end + describe "get_course_statistics/1" do + test "aggregates per-course llm stats and includes question-level breakdown" do + course = insert(:course) + + assessment = + insert(:assessment, + course: course, + is_published: true, + llm_total_input_tokens: 100, + llm_total_output_tokens: 200, + llm_total_cost: Decimal.new("1.50") + ) + + question_1 = + insert(:question, + assessment: assessment, + display_order: 1, + question: %{"llm_prompt" => "prompt"} + ) + + question_2 = + insert(:question, + assessment: assessment, + display_order: 2, + question: %{"llm_prompt" => "prompt2"} + ) + + student = insert(:course_registration, course: course, role: :student) + submission = insert(:submission, assessment: assessment, student: student) + answer1 = insert(:answer, submission: submission, question: question_1) + answer2 = insert(:answer, submission: submission, question: question_2) + + user = insert(:user) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + answer_id: answer1.id, + submission_id: submission.id, + user_id: user.id + }) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_2.id, + answer_id: answer2.id, + submission_id: submission.id, + user_id: user.id + }) + + assert {:ok, _} = + LLMStats.submit_feedback(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_1.id, + user_id: user.id, + rating: 4, + body: "good" + }) + + assert {:ok, _} = + LLMStats.submit_feedback(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question_2.id, + user_id: user.id, + rating: 2, + body: "bad" + }) + + result = LLMStats.get_course_statistics(course.id) + + assert result.course_total_input_tokens == 100 + assert result.course_total_output_tokens == 200 + assert Decimal.cmp(result.course_total_cost, Decimal.new("1.50")) == :eq + assert length(result.assessments) == 1 + + [assessment_stats] = result.assessments + assert assessment_stats.total_uses == 2 + assert assessment_stats.avg_rating == 3.0 + assert length(assessment_stats.questions) == 2 + end + + test "excludes assessments without llm_assessment_prompt and llm_prompt from course statistics" do + course = insert(:course) + + llm_assessment = + insert(:assessment, + course: course, + is_published: true, + title: "Mission With LLM", + llm_assessment_prompt: "Use this rubric", + llm_total_input_tokens: 10, + llm_total_output_tokens: 20, + llm_total_cost: Decimal.new("0.50") + ) + + # This assessment should never appear in course-level LLM stats. + insert(:assessment, + course: course, + is_published: true, + title: "Mission Without LLM", + llm_assessment_prompt: nil, + llm_total_input_tokens: 999, + llm_total_output_tokens: 999, + llm_total_cost: Decimal.new("9.99") + ) + + result = LLMStats.get_course_statistics(course.id) + + assert length(result.assessments) == 1 + [assessment_stats] = result.assessments + assert assessment_stats.assessment_id == llm_assessment.id + assert assessment_stats.title == "Mission With LLM" + assert result.course_total_input_tokens == 10 + assert result.course_total_output_tokens == 20 + assert Decimal.cmp(result.course_total_cost, Decimal.new("0.50")) == :eq + end + + test "includes assessments with question-level llm_prompt even when llm_assessment_prompt is nil" do + course = insert(:course) + + question_prompt_assessment = + insert(:assessment, + course: course, + is_published: true, + title: "Question Prompt Only", + llm_assessment_prompt: nil, + llm_total_input_tokens: 30, + llm_total_output_tokens: 40, + llm_total_cost: Decimal.new("0.70") + ) + + insert(:question, + assessment: question_prompt_assessment, + display_order: 1, + question: %{"llm_prompt" => "grade with rubric"} + ) + + insert(:assessment, + course: course, + is_published: true, + title: "No LLM Tags", + llm_assessment_prompt: nil, + llm_total_input_tokens: 999, + llm_total_output_tokens: 999, + llm_total_cost: Decimal.new("9.99") + ) + + result = LLMStats.get_course_statistics(course.id) + + assert length(result.assessments) == 1 + [assessment_stats] = result.assessments + assert assessment_stats.assessment_id == question_prompt_assessment.id + assert assessment_stats.title == "Question Prompt Only" + assert result.course_total_input_tokens == 30 + assert result.course_total_output_tokens == 40 + assert Decimal.cmp(result.course_total_cost, Decimal.new("0.70")) == :eq + end + end + describe "get_question_statistics/3" do test "returns statistics scoped to one question" do course = insert(:course) diff --git a/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs b/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs index 2095f4c5a..60bd4a0e6 100644 --- a/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs @@ -3,6 +3,84 @@ defmodule CadetWeb.AdminLLMStatsControllerTest do alias Cadet.{LLMStats, Repo, Courses.Course} + describe "GET /v2/courses/:course_id/admin/llm-stats" do + test "401 when not logged in", %{conn: conn} do + course = insert(:course) + + conn = get(conn, "/v2/courses/#{course.id}/admin/llm-stats") + assert response(conn, 401) =~ "Unauthorised" + end + + @tag authenticate: :student + test "403 for students", %{conn: conn} do + course_id = conn.assigns.course_id + insert(:assessment, course_id: course_id, is_published: true) + + conn = get(conn, "/v2/courses/#{course_id}/admin/llm-stats") + assert response(conn, 403) =~ "Forbidden" + end + + @tag authenticate: :staff + test "returns course-level llm stats and assessment breakdown", %{conn: conn} do + course = Repo.get!(Course, conn.assigns.course_id) + + assessment = + insert(:assessment, + course: course, + title: "Mission With LLM", + is_published: true, + llm_total_input_tokens: 10, + llm_total_output_tokens: 20, + llm_total_cost: Decimal.new("0.5") + ) + + insert(:assessment, + course: course, + title: "Mission Without LLM", + is_published: true, + llm_assessment_prompt: nil, + llm_total_input_tokens: 999, + llm_total_output_tokens: 999, + llm_total_cost: Decimal.new("9.99") + ) + + question = + insert(:question, + assessment: assessment, + display_order: 1, + question: %{"llm_prompt" => "x"} + ) + + student = insert(:course_registration, course: course, role: :student) + submission = insert(:submission, assessment: assessment, student: student) + answer = insert(:answer, submission: submission, question: question) + + assert {:ok, _} = + LLMStats.log_usage(%{ + course_id: course.id, + assessment_id: assessment.id, + question_id: question.id, + answer_id: answer.id, + submission_id: submission.id, + user_id: student.user_id + }) + + resp = + conn + |> get("/v2/courses/#{course.id}/admin/llm-stats") + |> json_response(200) + + assert resp["course_total_input_tokens"] == 10 + assert resp["course_total_output_tokens"] == 20 + assert resp["course_total_cost"] == "0.5" + assert length(resp["assessments"]) == 1 + [as] = resp["assessments"] + assert as["title"] == "Mission With LLM" + assert as["total_uses"] == 1 + assert length(as["questions"]) == 1 + end + end + describe "GET /v2/courses/:course_id/admin/llm-stats/:assessment_id" do test "401 when not logged in", %{conn: conn} do course = insert(:course) diff --git a/test/cadet_web/controllers/courses_controller_test.exs b/test/cadet_web/controllers/courses_controller_test.exs index 876c0166e..cb7d807b7 100644 --- a/test/cadet_web/controllers/courses_controller_test.exs +++ b/test/cadet_web/controllers/courses_controller_test.exs @@ -169,6 +169,7 @@ defmodule CadetWeb.CoursesControllerTest do "enableAchievements" => true, "enableSourcecast" => true, "enableStories" => false, + "hasLlmContent" => false, "sourceChapter" => 1, "sourceVariant" => "default", "moduleHelpText" => "Help Text", @@ -177,6 +178,44 @@ defmodule CadetWeb.CoursesControllerTest do } = resp end + @tag authenticate: :student + test "returns hasLlmContent true when assessment has non-empty llm_assessment_prompt", %{ + conn: conn + } do + course_id = conn.assigns[:course_id] + course = Repo.get(Course, course_id) + + insert(:assessment, course: course, llm_assessment_prompt: "Use this grading rubric") + + resp = conn |> get(build_url_config(course_id)) |> json_response(200) + + assert %{ + "config" => %{ + "hasLlmContent" => true + } + } = resp + end + + @tag authenticate: :student + test "returns hasLlmContent true when any question has non-empty llm_prompt", %{conn: conn} do + course_id = conn.assigns[:course_id] + course = Repo.get(Course, course_id) + assessment = insert(:assessment, course: course, llm_assessment_prompt: nil) + + insert(:question, + assessment: assessment, + question: build(:programming_question_content, llm_prompt: "Provide AI feedback") + ) + + resp = conn |> get(build_url_config(course_id)) |> json_response(200) + + assert %{ + "config" => %{ + "hasLlmContent" => true + } + } = resp + end + @tag authenticate: :student test "returns with error for user not belonging to the specified course", %{conn: conn} do course_id = conn.assigns[:course_id] diff --git a/test/cadet_web/controllers/user_controller_test.exs b/test/cadet_web/controllers/user_controller_test.exs index 4cf471297..7084d08b9 100644 --- a/test/cadet_web/controllers/user_controller_test.exs +++ b/test/cadet_web/controllers/user_controller_test.exs @@ -108,6 +108,8 @@ defmodule CadetWeb.UserControllerTest do "enableGame" => true, "enableSourcecast" => true, "enableStories" => false, + "enableLlmGrading" => false, + "hasLlmContent" => false, "courseShortName" => "CS1101S", "moduleHelpText" => "Help Text", "courseName" => "Programming Methodology", @@ -325,8 +327,10 @@ defmodule CadetWeb.UserControllerTest do "enableAchievements" => true, "enableGame" => true, "enableSourcecast" => true, + "enableLlmGrading" => false, "courseShortName" => "CS1101S", "enableStories" => false, + "hasLlmContent" => false, "moduleHelpText" => "Help Text", "courseName" => "Programming Methodology", "sourceChapter" => 1, @@ -344,6 +348,26 @@ defmodule CadetWeb.UserControllerTest do assert expected == resp end + @tag authenticate: :student + test "includes hasLlmContent when latest viewed course contains llm-tagged assessment", %{ + conn: conn + } do + course = conn.assigns.current_user.latest_viewed_course + + insert(:assessment, %{ + is_published: true, + course: course, + llm_assessment_prompt: "Use this rubric" + }) + + resp = + conn + |> get("/v2/user/latest_viewed_course") + |> json_response(200) + + assert resp["courseConfiguration"]["hasLlmContent"] == true + end + @tag sign_in: %{latest_viewed_course: nil} test "success, no latest_viewed_course", %{conn: conn} do resp = From a9d1a8beb51e97cf644e3a86af2b355b6b1fe427 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 12:51:49 +0800 Subject: [PATCH 53/68] fixed possible type mismatch --- lib/cadet/llm_stats.ex | 292 +++++++++++++++++----------------- test/cadet/llm_stats_test.exs | 23 +++ 2 files changed, 168 insertions(+), 147 deletions(-) diff --git a/lib/cadet/llm_stats.ex b/lib/cadet/llm_stats.ex index 97fc9b951..7f5e6437f 100644 --- a/lib/cadet/llm_stats.ex +++ b/lib/cadet/llm_stats.ex @@ -6,6 +6,7 @@ defmodule Cadet.LLMStats do import Ecto.Query alias Cadet.Repo + alias Cadet.Assessments.{Assessment, Question} alias Cadet.LLMStats.{LLMUsageLog, LLMFeedback} # ===================== @@ -60,7 +61,6 @@ defmodule Cadet.LLMStats do ) ) - # Per-question breakdown questions = Repo.all( from(l in LLMUsageLog, @@ -78,10 +78,9 @@ defmodule Cadet.LLMStats do ) ) - # ADDED: Fetch the cost and token data from the Assessment table costs = Repo.one( - from(a in Cadet.Assessments.Assessment, + from(a in Assessment, where: a.id == ^assessment_id and a.course_id == ^course_id, select: %{ llm_total_cost: a.llm_total_cost, @@ -92,13 +91,11 @@ defmodule Cadet.LLMStats do ) ) || %{} - # Merge the costs into the final map that gets sent to React %{ total_uses: total_uses, unique_submissions: unique_submissions, unique_users: unique_users, questions: questions, - # Add the cost data (with safe fallbacks if nil) llm_total_cost: Map.get(costs, :llm_total_cost) || Decimal.new("0.0"), llm_total_input_tokens: Map.get(costs, :llm_total_input_tokens) || 0, llm_total_output_tokens: Map.get(costs, :llm_total_output_tokens) || 0, @@ -107,8 +104,21 @@ defmodule Cadet.LLMStats do end def get_course_statistics(course_id) do - assessments = - from(a in Cadet.Assessments.Assessment, + assessments_with_stats = + fetch_llm_course_assessments(course_id) + |> Enum.map(&build_course_assessment_stats(course_id, &1)) + + %{ + course_total_input_tokens: sum_assessment_input_tokens(assessments_with_stats), + course_total_output_tokens: sum_assessment_output_tokens(assessments_with_stats), + course_total_cost: sum_assessment_costs(assessments_with_stats), + assessments: assessments_with_stats + } + end + + defp fetch_llm_course_assessments(course_id) do + Repo.all( + from(a in Assessment, where: a.course_id == ^course_id and a.is_published == true, where: fragment("? IS NOT NULL AND ? != ''", a.llm_assessment_prompt, a.llm_assessment_prompt) or @@ -123,157 +133,145 @@ defmodule Cadet.LLMStats do category: c.type, llm_total_input_tokens: coalesce(a.llm_total_input_tokens, 0), llm_total_output_tokens: coalesce(a.llm_total_output_tokens, 0), - llm_total_cost: coalesce(a.llm_total_cost, 0) + llm_total_cost: coalesce(a.llm_total_cost, type(^Decimal.new("0.0"), :decimal)) } ) - |> Repo.all() + ) + end - assessments_with_stats = - Enum.map(assessments, fn assessment -> - total_uses = - Repo.one( - from(l in LLMUsageLog, - where: l.course_id == ^course_id and l.assessment_id == ^assessment.assessment_id, - select: count(l.id) - ) - ) || 0 - - avg_rating = - Repo.one( - from(f in LLMFeedback, - where: - f.course_id == ^course_id and f.assessment_id == ^assessment.assessment_id and - not is_nil(f.rating), - select: avg(f.rating) - ) - ) - - avg_rating = - if is_nil(avg_rating) do - nil - else - avg_rating |> Decimal.to_float() |> Float.round(2) - end - - questions = - Repo.all( - from(q in Cadet.Assessments.Question, - where: q.assessment_id == ^assessment.assessment_id, - where: - fragment( - "? ->> 'llm_prompt' IS NOT NULL AND ? ->> 'llm_prompt' != ''", - q.question, - q.question - ), - order_by: [asc: q.display_order], - select: %{ - question_id: q.id, - display_order: q.display_order - } - ) - ) - - question_stats = - Enum.map(questions, fn question -> - question_uses = - Repo.one( - from(l in LLMUsageLog, - where: - l.course_id == ^course_id and l.assessment_id == ^assessment.assessment_id and - l.question_id == ^question.question_id, - select: count(l.id) - ) - ) || 0 - - question_rating = - Repo.one( - from(f in LLMFeedback, - where: - f.course_id == ^course_id and f.assessment_id == ^assessment.assessment_id and - f.question_id == ^question.question_id and not is_nil(f.rating), - select: avg(f.rating) - ) - ) - - question_rating = - if is_nil(question_rating) do - nil - else - question_rating |> Decimal.to_float() |> Float.round(2) - end - - question_input_tokens = - if total_uses > 0 do - round(assessment.llm_total_input_tokens * question_uses / total_uses) - else - 0 - end - - question_output_tokens = - if total_uses > 0 do - round(assessment.llm_total_output_tokens * question_uses / total_uses) - else - 0 - end - - question_cost = - if total_uses > 0 do - Decimal.mult( - assessment.llm_total_cost, - Decimal.div(Decimal.new(question_uses), Decimal.new(total_uses)) - ) - |> Decimal.round(6, :half_up) - else - Decimal.new("0.0") - end - - %{ - question_id: question.question_id, - display_order: question.display_order, - total_uses: question_uses, - avg_rating: question_rating, - llm_total_input_tokens: question_input_tokens, - llm_total_output_tokens: question_output_tokens, - llm_total_cost: question_cost - } - end) - - %{ - assessment_id: assessment.assessment_id, - title: assessment.title, - category: assessment.category, - total_uses: total_uses, - avg_rating: avg_rating, - llm_total_input_tokens: assessment.llm_total_input_tokens, - llm_total_output_tokens: assessment.llm_total_output_tokens, - llm_total_cost: assessment.llm_total_cost, - questions: question_stats - } - end) + defp build_course_assessment_stats(course_id, assessment) do + total_uses = get_assessment_total_uses(course_id, assessment.assessment_id) - course_total_input_tokens = - Enum.reduce(assessments_with_stats, 0, fn assessment, acc -> - acc + assessment.llm_total_input_tokens - end) + %{ + assessment_id: assessment.assessment_id, + title: assessment.title, + category: assessment.category, + total_uses: total_uses, + avg_rating: get_assessment_avg_rating(course_id, assessment.assessment_id), + llm_total_input_tokens: assessment.llm_total_input_tokens, + llm_total_output_tokens: assessment.llm_total_output_tokens, + llm_total_cost: assessment.llm_total_cost, + questions: get_question_stats(course_id, assessment, total_uses) + } + end - course_total_output_tokens = - Enum.reduce(assessments_with_stats, 0, fn assessment, acc -> - acc + assessment.llm_total_output_tokens - end) + defp get_assessment_total_uses(course_id, assessment_id) do + Repo.one( + from(l in LLMUsageLog, + where: l.course_id == ^course_id and l.assessment_id == ^assessment_id, + select: count(l.id) + ) + ) || 0 + end - course_total_cost = - Enum.reduce(assessments_with_stats, Decimal.new("0.0"), fn assessment, acc -> - Decimal.add(acc, assessment.llm_total_cost) - end) + defp get_assessment_avg_rating(course_id, assessment_id) do + Repo.one( + from(f in LLMFeedback, + where: f.course_id == ^course_id and f.assessment_id == ^assessment_id, + where: not is_nil(f.rating), + select: avg(f.rating) + ) + ) + |> normalize_avg_rating() + end + + defp get_llm_questions(assessment_id) do + Repo.all( + from(q in Question, + where: q.assessment_id == ^assessment_id, + where: + fragment( + "? ->> 'llm_prompt' IS NOT NULL AND ? ->> 'llm_prompt' != ''", + q.question, + q.question + ), + order_by: [asc: q.display_order], + select: %{question_id: q.id, display_order: q.display_order} + ) + ) + end + + defp get_question_stats(course_id, assessment, total_uses) do + get_llm_questions(assessment.assessment_id) + |> Enum.map(&build_question_stats(course_id, assessment, total_uses, &1)) + end + + defp build_question_stats(course_id, assessment, total_uses, question) do + question_uses = + get_question_total_uses(course_id, assessment.assessment_id, question.question_id) %{ - course_total_input_tokens: course_total_input_tokens, - course_total_output_tokens: course_total_output_tokens, - course_total_cost: course_total_cost, - assessments: assessments_with_stats + question_id: question.question_id, + display_order: question.display_order, + total_uses: question_uses, + avg_rating: + get_question_avg_rating(course_id, assessment.assessment_id, question.question_id), + llm_total_input_tokens: + proportional_token_count(assessment.llm_total_input_tokens, question_uses, total_uses), + llm_total_output_tokens: + proportional_token_count(assessment.llm_total_output_tokens, question_uses, total_uses), + llm_total_cost: proportional_cost(assessment.llm_total_cost, question_uses, total_uses) } end + defp get_question_total_uses(course_id, assessment_id, question_id) do + Repo.one( + from(l in LLMUsageLog, + where: + l.course_id == ^course_id and l.assessment_id == ^assessment_id and + l.question_id == ^question_id, + select: count(l.id) + ) + ) || 0 + end + + defp get_question_avg_rating(course_id, assessment_id, question_id) do + Repo.one( + from(f in LLMFeedback, + where: + f.course_id == ^course_id and f.assessment_id == ^assessment_id and + f.question_id == ^question_id, + where: not is_nil(f.rating), + select: avg(f.rating) + ) + ) + |> normalize_avg_rating() + end + + defp normalize_avg_rating(nil), do: nil + defp normalize_avg_rating(avg_rating), do: Float.round(Decimal.to_float(avg_rating), 2) + + defp proportional_token_count(_total_tokens, _question_uses, 0), do: 0 + + defp proportional_token_count(total_tokens, question_uses, total_uses) do + round(total_tokens * question_uses / total_uses) + end + + defp proportional_cost(_total_cost, _question_uses, 0), do: Decimal.new("0.0") + + defp proportional_cost(total_cost, question_uses, total_uses) do + cost_fraction = Decimal.div(Decimal.new(question_uses), Decimal.new(total_uses)) + Decimal.round(Decimal.mult(total_cost, cost_fraction), 6, :half_up) + end + + defp sum_assessment_input_tokens(assessments_with_stats) do + Enum.reduce(assessments_with_stats, 0, fn assessment, acc -> + acc + assessment.llm_total_input_tokens + end) + end + + defp sum_assessment_output_tokens(assessments_with_stats) do + Enum.reduce(assessments_with_stats, 0, fn assessment, acc -> + acc + assessment.llm_total_output_tokens + end) + end + + defp sum_assessment_costs(assessments_with_stats) do + Enum.reduce(assessments_with_stats, Decimal.new("0.0"), fn assessment, acc -> + Decimal.add(acc, assessment.llm_total_cost) + end) + end + # ===================== # Question-level Statistics # ===================== diff --git a/test/cadet/llm_stats_test.exs b/test/cadet/llm_stats_test.exs index c6a315f62..0c2f77a53 100644 --- a/test/cadet/llm_stats_test.exs +++ b/test/cadet/llm_stats_test.exs @@ -296,6 +296,29 @@ defmodule Cadet.LLMStatsTest do assert result.course_total_output_tokens == 40 assert Decimal.cmp(result.course_total_cost, Decimal.new("0.70")) == :eq end + + test "treats null llm_total_cost as Decimal zero for course statistics" do + course = insert(:course) + + assessment = + insert(:assessment, + course: course, + is_published: true, + title: "Null Cost Assessment", + llm_assessment_prompt: "Use this rubric", + llm_total_input_tokens: 10, + llm_total_output_tokens: 20, + llm_total_cost: nil + ) + + result = LLMStats.get_course_statistics(course.id) + + assert length(result.assessments) == 1 + [assessment_stats] = result.assessments + assert assessment_stats.assessment_id == assessment.id + assert Decimal.cmp(assessment_stats.llm_total_cost, Decimal.new("0.0")) == :eq + assert Decimal.cmp(result.course_total_cost, Decimal.new("0.0")) == :eq + end end describe "get_question_statistics/3" do From c011c040789383eea67ce9a13383b5062aced517 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 14:46:31 +0800 Subject: [PATCH 54/68] fixed potential silent data logging failure --- lib/cadet/llm_stats.ex | 45 ++++++++++--------- .../controllers/generate_ai_comments.ex | 23 ++++++++-- .../ai_code_analysis_controller_test.exs | 14 ++++++ 3 files changed, 57 insertions(+), 25 deletions(-) diff --git a/lib/cadet/llm_stats.ex b/lib/cadet/llm_stats.ex index 7f5e6437f..4fce20e3f 100644 --- a/lib/cadet/llm_stats.ex +++ b/lib/cadet/llm_stats.ex @@ -104,9 +104,8 @@ defmodule Cadet.LLMStats do end def get_course_statistics(course_id) do - assessments_with_stats = - fetch_llm_course_assessments(course_id) - |> Enum.map(&build_course_assessment_stats(course_id, &1)) + assessments = fetch_llm_course_assessments(course_id) + assessments_with_stats = Enum.map(assessments, &build_course_assessment_stats(course_id, &1)) %{ course_total_input_tokens: sum_assessment_input_tokens(assessments_with_stats), @@ -165,14 +164,16 @@ defmodule Cadet.LLMStats do end defp get_assessment_avg_rating(course_id, assessment_id) do - Repo.one( - from(f in LLMFeedback, - where: f.course_id == ^course_id and f.assessment_id == ^assessment_id, - where: not is_nil(f.rating), - select: avg(f.rating) + avg_rating = + Repo.one( + from(f in LLMFeedback, + where: f.course_id == ^course_id and f.assessment_id == ^assessment_id, + where: not is_nil(f.rating), + select: avg(f.rating) + ) ) - ) - |> normalize_avg_rating() + + normalize_avg_rating(avg_rating) end defp get_llm_questions(assessment_id) do @@ -192,8 +193,8 @@ defmodule Cadet.LLMStats do end defp get_question_stats(course_id, assessment, total_uses) do - get_llm_questions(assessment.assessment_id) - |> Enum.map(&build_question_stats(course_id, assessment, total_uses, &1)) + llm_questions = get_llm_questions(assessment.assessment_id) + Enum.map(llm_questions, &build_question_stats(course_id, assessment, total_uses, &1)) end defp build_question_stats(course_id, assessment, total_uses, question) do @@ -226,16 +227,18 @@ defmodule Cadet.LLMStats do end defp get_question_avg_rating(course_id, assessment_id, question_id) do - Repo.one( - from(f in LLMFeedback, - where: - f.course_id == ^course_id and f.assessment_id == ^assessment_id and - f.question_id == ^question_id, - where: not is_nil(f.rating), - select: avg(f.rating) + avg_rating = + Repo.one( + from(f in LLMFeedback, + where: + f.course_id == ^course_id and f.assessment_id == ^assessment_id and + f.question_id == ^question_id, + where: not is_nil(f.rating), + select: avg(f.rating) + ) ) - ) - |> normalize_avg_rating() + + normalize_avg_rating(avg_rating) end defp normalize_avg_rating(nil), do: nil diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 103d9e384..2b8b3416c 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -79,8 +79,9 @@ defmodule CadetWeb.AICodeAnalysisController do "course_id" => course_id }) when is_ecto_id(answer_id) do - with {answer_id_parsed, ""} <- Integer.parse(answer_id), - {:ok, course} <- Courses.get_course_config(course_id), + with {:ok, answer_id_parsed} <- parse_answer_id(answer_id), + {:ok, course_id_parsed} <- parse_course_id(course_id), + {:ok, course} <- Courses.get_course_config(course_id_parsed), {:ok} <- ensure_llm_enabled(course), {:ok, key} <- AICommentsHelpers.decrypt_llm_api_key(course.llm_api_key), {:ok} <- @@ -102,15 +103,20 @@ defmodule CadetWeb.AICodeAnalysisController do llm_api_url: course.llm_api_url, course_prompt: course.llm_course_level_prompt, assessment_prompt: Assessments.get_llm_assessment_prompt(answer.question_id), - course_id: course_id + course_id: course_id_parsed } ) else - :error -> + {:error, :invalid_answer_id} -> conn |> put_status(:bad_request) |> text("Invalid question ID format") + {:error, :invalid_course_id} -> + conn + |> put_status(:bad_request) + |> text("Invalid course ID format") + {:decrypt_error, err} -> conn |> put_status(:internal_server_error) @@ -382,6 +388,15 @@ defmodule CadetWeb.AICodeAnalysisController do end end + defp parse_course_id(course_id) when is_integer(course_id), do: {:ok, course_id} + + defp parse_course_id(course_id) when is_binary(course_id) do + case Integer.parse(course_id) do + {parsed, ""} -> {:ok, parsed} + _ -> {:error, :invalid_course_id} + end + end + defp parse_edits(edits) when is_map(edits) do edits |> Enum.reduce_while({:ok, []}, fn {index_str, edited_text}, {:ok, acc} -> diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index c6e1ca3a4..a8eda8e11 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -105,6 +105,20 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do end end + test "errors out when given an invalid course id", %{ + conn: conn, + admin_user: admin_user, + answer: answer + } do + response = + conn + |> sign_in(admin_user.user) + |> post(build_url_generate_ai_comments("invalid-course-id", answer.id)) + |> text_response(400) + + assert response == "Invalid course ID format" + end + test "LLM endpoint returns an invalid response - should log errors in database", %{ conn: conn, admin_user: admin_user, From 304e633f682228360707477898ae2e38275b8fc7 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 15:02:39 +0800 Subject: [PATCH 55/68] fixed source errors --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b988a2c3..9004e1667 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,8 @@ jobs: options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - uses: actions/checkout@v6 + with: + fetch-depth: 0 - name: Cache deps uses: actions/cache@v5 with: From 9721dc7e2c48d0af33d8afc87c2d8bbf5f69f376 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 15:17:24 +0800 Subject: [PATCH 56/68] fixed MCQ question embed handling --- lib/cadet/ai_comments.ex | 5 ++++- lib/cadet/assessments/question_types/mcq_question.ex | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 7ca71234b..ce8979e42 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -85,7 +85,10 @@ defmodule Cadet.AIComments do Repo.transaction(fn -> # Serialize version creation per (ai_comment_id, comment_index) # to avoid duplicate version numbers. - case Repo.query("SELECT pg_advisory_xact_lock($1, $2)", [ai_comment_id, comment_index]) do + case Repo.query( + "SELECT pg_advisory_xact_lock((($1::bigint << 32) | $2::bigint)::bigint)", + [ai_comment_id, comment_index] + ) do {:ok, _} -> next_version = Repo.one( diff --git a/lib/cadet/assessments/question_types/mcq_question.ex b/lib/cadet/assessments/question_types/mcq_question.ex index b99a5031f..f6ede3a73 100644 --- a/lib/cadet/assessments/question_types/mcq_question.ex +++ b/lib/cadet/assessments/question_types/mcq_question.ex @@ -20,7 +20,7 @@ defmodule Cadet.Assessments.QuestionTypes.MCQQuestion do |> cast(params, @required_fields) |> cast_embed(:choices, with: &MCQChoice.changeset/2, required: true) |> validate_one_correct_answer - |> validate_required(@required_fields ++ ~w(choices)a) + |> validate_required(@required_fields) end defp validate_one_correct_answer(changeset) do From 0f4a7d4fffdba3b6c780494c7645b08c9e42292c Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 15:31:07 +0800 Subject: [PATCH 57/68] fix invalid cours id crash path --- lib/cadet_web/router.ex | 15 ++++++++++++++- test/cadet/accounts/course_registration_test.exs | 10 ++++++---- test/cadet/assessments/assessments_test.exs | 12 ++++++------ test/cadet/llm_stats_test.exs | 10 +++++----- .../admin_llm_stats_controller_test.exs | 5 +++-- .../ai_code_analysis_controller_test.exs | 8 ++++---- .../controllers/assessments_controller_test.exs | 4 ++-- 7 files changed, 40 insertions(+), 24 deletions(-) diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index 9509cd6e7..3f156adf5 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -320,8 +320,21 @@ defmodule CadetWeb.Router do defp assign_course(conn, _opts) do course_id = conn.path_params["course_id"] + parsed_course_id = + case Integer.parse(to_string(course_id)) do + {id, ""} -> id + _ -> nil + end + course_reg = - Cadet.Accounts.CourseRegistrations.get_user_record(conn.assigns.current_user.id, course_id) + if is_nil(parsed_course_id) do + nil + else + Cadet.Accounts.CourseRegistrations.get_user_record( + conn.assigns.current_user.id, + parsed_course_id + ) + end case course_reg do nil -> conn |> send_resp(403, "Forbidden") |> halt() diff --git a/test/cadet/accounts/course_registration_test.exs b/test/cadet/accounts/course_registration_test.exs index 0fec0d858..ea1b1c538 100644 --- a/test/cadet/accounts/course_registration_test.exs +++ b/test/cadet/accounts/course_registration_test.exs @@ -286,10 +286,12 @@ defmodule Cadet.Accounts.CourseRegistrationTest do assert length(CourseRegistrations.get_users(course1.id)) == 1 assert_raise FunctionClauseError, fn -> - CourseRegistrations.insert_or_update_course_registration(%{ - user_id: user2.id, - course_id: course1.id - }) + apply(CourseRegistrations, :insert_or_update_course_registration, [ + %{ + user_id: user2.id, + course_id: course1.id + } + ]) end assert length(CourseRegistrations.get_users(course1.id)) == 1 diff --git a/test/cadet/assessments/assessments_test.exs b/test/cadet/assessments/assessments_test.exs index 5594e8ff2..ba7744a45 100644 --- a/test/cadet/assessments/assessments_test.exs +++ b/test/cadet/assessments/assessments_test.exs @@ -200,7 +200,7 @@ defmodule Cadet.AssessmentsTest do assert updated.llm_total_input_tokens == 10 assert updated.llm_total_output_tokens == 20 assert updated.llm_total_cached_tokens == 5 - assert Decimal.cmp(updated.llm_total_cost, Decimal.new("0.000288")) == :eq + assert Decimal.compare(updated.llm_total_cost, Decimal.new("0.000288")) == :eq end test "force update assessment with invalid params" do @@ -3225,12 +3225,12 @@ defmodule Cadet.AssessmentsTest do test "correctly fetches all students with their xp in descending order", %{course: course} do all_user_xp = Assessments.all_user_total_xp(course.id) - assert get_all_student_xp(all_user_xp) == 50..1 |> Enum.to_list() + assert get_all_student_xp(all_user_xp) == 50..1//-1 |> Enum.to_list() end test "correctly fetches only relevant students for leaderboard display with potential overflow", %{course: course} do - Enum.each(1..50, fn x -> + Enum.each(1..50, fn _x -> offset = Enum.random(0..49) limit = Enum.random(1..50) @@ -3238,7 +3238,7 @@ defmodule Cadet.AssessmentsTest do Assessments.all_user_total_xp(course.id, %{offset: offset, limit: limit}) expected_xp_list = - 50..1 + 50..1//-1 |> Enum.to_list() |> Enum.slice(offset, limit) @@ -3298,7 +3298,7 @@ defmodule Cadet.AssessmentsTest do fn student -> Enum.map( Enum.with_index(submission_list), - fn {submission, index} -> + fn {submission, _index} -> insert( :submission_vote, voter: student, @@ -3395,7 +3395,7 @@ defmodule Cadet.AssessmentsTest do test "correctly assigns xp to winning contest entries with defined xp values", %{ course: course, voting_question: voting_question, - ans_list: ans_list + ans_list: _ans_list } do # update defined xp_values for voting question Question diff --git a/test/cadet/llm_stats_test.exs b/test/cadet/llm_stats_test.exs index 0c2f77a53..12d35d2ca 100644 --- a/test/cadet/llm_stats_test.exs +++ b/test/cadet/llm_stats_test.exs @@ -211,7 +211,7 @@ defmodule Cadet.LLMStatsTest do assert result.course_total_input_tokens == 100 assert result.course_total_output_tokens == 200 - assert Decimal.cmp(result.course_total_cost, Decimal.new("1.50")) == :eq + assert Decimal.compare(result.course_total_cost, Decimal.new("1.50")) == :eq assert length(result.assessments) == 1 [assessment_stats] = result.assessments @@ -253,7 +253,7 @@ defmodule Cadet.LLMStatsTest do assert assessment_stats.title == "Mission With LLM" assert result.course_total_input_tokens == 10 assert result.course_total_output_tokens == 20 - assert Decimal.cmp(result.course_total_cost, Decimal.new("0.50")) == :eq + assert Decimal.compare(result.course_total_cost, Decimal.new("0.50")) == :eq end test "includes assessments with question-level llm_prompt even when llm_assessment_prompt is nil" do @@ -294,7 +294,7 @@ defmodule Cadet.LLMStatsTest do assert assessment_stats.title == "Question Prompt Only" assert result.course_total_input_tokens == 30 assert result.course_total_output_tokens == 40 - assert Decimal.cmp(result.course_total_cost, Decimal.new("0.70")) == :eq + assert Decimal.compare(result.course_total_cost, Decimal.new("0.70")) == :eq end test "treats null llm_total_cost as Decimal zero for course statistics" do @@ -316,8 +316,8 @@ defmodule Cadet.LLMStatsTest do assert length(result.assessments) == 1 [assessment_stats] = result.assessments assert assessment_stats.assessment_id == assessment.id - assert Decimal.cmp(assessment_stats.llm_total_cost, Decimal.new("0.0")) == :eq - assert Decimal.cmp(result.course_total_cost, Decimal.new("0.0")) == :eq + assert Decimal.compare(assessment_stats.llm_total_cost, Decimal.new("0.0")) == :eq + assert Decimal.compare(result.course_total_cost, Decimal.new("0.0")) == :eq end end diff --git a/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs b/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs index 60bd4a0e6..f8d64434d 100644 --- a/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_llm_stats_controller_test.exs @@ -14,7 +14,8 @@ defmodule CadetWeb.AdminLLMStatsControllerTest do @tag authenticate: :student test "403 for students", %{conn: conn} do course_id = conn.assigns.course_id - insert(:assessment, course_id: course_id, is_published: true) + course = Repo.get!(Course, course_id) + insert(:assessment, course: course, is_published: true) conn = get(conn, "/v2/courses/#{course_id}/admin/llm-stats") assert response(conn, 403) =~ "Forbidden" @@ -72,7 +73,7 @@ defmodule CadetWeb.AdminLLMStatsControllerTest do assert resp["course_total_input_tokens"] == 10 assert resp["course_total_output_tokens"] == 20 - assert resp["course_total_cost"] == "0.5" + assert resp["course_total_cost"] == "0.500000" assert length(resp["assessments"]) == 1 [as] = resp["assessments"] assert as["title"] == "Mission With LLM" diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index a8eda8e11..c379fe916 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -2,8 +2,8 @@ import Mock defmodule CadetWeb.AICodeAnalysisControllerTest do use CadetWeb.ConnCase - alias Cadet.{Repo, AIComments} - alias Cadet.{AIComments.AIComment, Courses.Course} + alias Cadet.Repo + alias Cadet.AIComments.AIComment alias CadetWeb.AICommentsHelpers setup do @@ -114,9 +114,9 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do conn |> sign_in(admin_user.user) |> post(build_url_generate_ai_comments("invalid-course-id", answer.id)) - |> text_response(400) + |> text_response(403) - assert response == "Invalid course ID format" + assert response == "Forbidden" end test "LLM endpoint returns an invalid response - should log errors in database", %{ diff --git a/test/cadet_web/controllers/assessments_controller_test.exs b/test/cadet_web/controllers/assessments_controller_test.exs index f037b0292..cb8ddfd1a 100644 --- a/test/cadet_web/controllers/assessments_controller_test.exs +++ b/test/cadet_web/controllers/assessments_controller_test.exs @@ -1972,7 +1972,7 @@ defmodule CadetWeb.AssessmentsControllerTest do defp build_url_unlock(course_id, assessment_id), do: "/v2/courses/#{course_id}/assessments/#{assessment_id}/unlock" - defp build_popular_leaderboard_url(course_id, assessment_id, params \\ %{}) do + defp build_popular_leaderboard_url(course_id, assessment_id, params) do base_url = "#{build_url(course_id, assessment_id)}/contest_popular_leaderboard" if params != %{} do @@ -1983,7 +1983,7 @@ defmodule CadetWeb.AssessmentsControllerTest do end end - defp build_score_leaderboard_url(course_id, assessment_id, params \\ %{}) do + defp build_score_leaderboard_url(course_id, assessment_id, params) do base_url = "#{build_url(course_id, assessment_id)}/contest_score_leaderboard" if params != %{} do From 043280ddace43f64dac1e44229b62ea351758973 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 15:35:58 +0800 Subject: [PATCH 58/68] fixed formatting --- test/cadet/accounts/course_registration_test.exs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test/cadet/accounts/course_registration_test.exs b/test/cadet/accounts/course_registration_test.exs index ea1b1c538..858fa7518 100644 --- a/test/cadet/accounts/course_registration_test.exs +++ b/test/cadet/accounts/course_registration_test.exs @@ -285,13 +285,11 @@ defmodule Cadet.Accounts.CourseRegistrationTest do test "failed due to incomplete changeset", %{course1: course1, user2: user2} do assert length(CourseRegistrations.get_users(course1.id)) == 1 + invalid_params = + for {k, v} <- [user_id: user2.id, course_id: course1.id], into: %{}, do: {k, v} + assert_raise FunctionClauseError, fn -> - apply(CourseRegistrations, :insert_or_update_course_registration, [ - %{ - user_id: user2.id, - course_id: course1.id - } - ]) + CourseRegistrations.insert_or_update_course_registration(invalid_params) end assert length(CourseRegistrations.get_users(course1.id)) == 1 From 40f23a9c1b56f4cbb9d6c4cda34c3b8ea07ab92d Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 15:51:39 +0800 Subject: [PATCH 59/68] fixed ai code analysis test file --- .../ai_code_analysis_controller_test.exs | 47 +++++++++---------- test/cadet_web/plug/rate_limiter_test.exs | 1 - 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index c379fe916..cb1ca9af7 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -41,9 +41,9 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do admin_user: admin_user, staff_user: staff_user, course_with_llm: course_with_llm, - example_assessment: example_assessment, - new_submission: new_submission, - question: question, + example_assessment: _example_assessment, + new_submission: _new_submission, + question: _question, answer: answer } do # Make the API call @@ -83,12 +83,12 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do test "errors out when given an invalid answer id", %{ conn: conn, admin_user: admin_user, - staff_user: staff_user, + staff_user: _staff_user, course_with_llm: course_with_llm, - example_assessment: example_assessment, - new_submission: new_submission, - question: question, - answer: answer + example_assessment: _example_assessment, + new_submission: _new_submission, + question: _question, + answer: _answer } do random_answer_id = 324_324 @@ -97,11 +97,10 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do chat_completion: fn _input, _overrides -> {:ok, %{:choices => [%{"message" => %{"content" => "Comment1|||Comment2"}}]}} end do - response = - conn - |> sign_in(admin_user.user) - |> post(build_url_generate_ai_comments(course_with_llm.id, random_answer_id)) - |> text_response(400) + conn + |> sign_in(admin_user.user) + |> post(build_url_generate_ai_comments(course_with_llm.id, random_answer_id)) + |> text_response(400) end end @@ -110,23 +109,22 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do admin_user: admin_user, answer: answer } do - response = + conn = conn |> sign_in(admin_user.user) |> post(build_url_generate_ai_comments("invalid-course-id", answer.id)) - |> text_response(403) - assert response == "Forbidden" + assert response(conn, 403) == "Forbidden" end test "LLM endpoint returns an invalid response - should log errors in database", %{ conn: conn, admin_user: admin_user, - staff_user: staff_user, + staff_user: _staff_user, course_with_llm: course_with_llm, - example_assessment: example_assessment, - new_submission: new_submission, - question: question, + example_assessment: _example_assessment, + new_submission: _new_submission, + question: _question, answer: answer } do # Make the API call that should fail @@ -134,11 +132,10 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do chat_completion: fn _input, _overrides -> {:ok, %{"body" => "Some unexpected response"}} end do - response = - conn - |> sign_in(admin_user.user) - |> post(build_url_generate_ai_comments(course_with_llm.id, answer.id)) - |> text_response(502) + conn + |> sign_in(admin_user.user) + |> post(build_url_generate_ai_comments(course_with_llm.id, answer.id)) + |> text_response(502) end # Verify database entry even with error diff --git a/test/cadet_web/plug/rate_limiter_test.exs b/test/cadet_web/plug/rate_limiter_test.exs index d5337f711..305d94d9a 100644 --- a/test/cadet_web/plug/rate_limiter_test.exs +++ b/test/cadet_web/plug/rate_limiter_test.exs @@ -1,6 +1,5 @@ defmodule CadetWeb.Plugs.RateLimiterTest do use CadetWeb.ConnCase - import Plug.Conn alias CadetWeb.Plugs.RateLimiter setup %{conn: conn} do From 1403530f6657fe72a5ef8dd7be93325c9461b681 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 16:44:57 +0800 Subject: [PATCH 60/68] fixed pattern match error --- lib/cadet/courses/courses.ex | 2 +- .../admin_grading_controller.ex | 5 +++++ .../controllers/courses_controller.ex | 5 +++++ .../controllers/generate_ai_comments.ex | 20 +++++++++---------- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/cadet/courses/courses.ex b/lib/cadet/courses/courses.ex index fe483883b..f2d1c8077 100644 --- a/lib/cadet/courses/courses.ex +++ b/lib/cadet/courses/courses.ex @@ -59,7 +59,7 @@ defmodule Cadet.Courses do @doc """ Returns the course configuration for the specified course. """ - @spec get_course_config(integer) :: + @spec get_course_config(integer | binary) :: {:ok, Course.t()} | {:error, {:bad_request, String.t()}} def get_course_config(course_id) when is_ecto_id(course_id) do Logger.info("Retrieving course configuration for course #{course_id}") diff --git a/lib/cadet_web/admin_controllers/admin_grading_controller.ex b/lib/cadet_web/admin_controllers/admin_grading_controller.ex index 564988b9f..3f9a52e8a 100644 --- a/lib/cadet_web/admin_controllers/admin_grading_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_grading_controller.ex @@ -2,6 +2,11 @@ defmodule CadetWeb.AdminGradingController do use CadetWeb, :controller use PhoenixSwagger + # Dialyzer false positive: Dialyzer cannot infer that assessment.course_id is guaranteed + # to be a positive integer, so it pessimistically assumes get_course_config may only + # return error. The code is safe because it handles both cases properly. + @dialyzer {:no_match, {:show, 2}} + alias Cadet.{Assessments, Courses} @doc """ diff --git a/lib/cadet_web/controllers/courses_controller.ex b/lib/cadet_web/controllers/courses_controller.ex index 3ad4c415d..59ee24c0d 100644 --- a/lib/cadet_web/controllers/courses_controller.ex +++ b/lib/cadet_web/controllers/courses_controller.ex @@ -4,6 +4,11 @@ defmodule CadetWeb.CoursesController do use PhoenixSwagger require Logger + # Dialyzer false positive: Dialyzer cannot infer that course_id is guaranteed to be + # a positive integer, so it pessimistically assumes get_course_config may only return + # error. The code is safe because it handles both cases properly. + @dialyzer {:no_match, {:index, 2}} + alias Cadet.Courses alias Cadet.Accounts.CourseRegistrations diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 2b8b3416c..a38fe5b0d 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -4,6 +4,15 @@ defmodule CadetWeb.AICodeAnalysisController do require HTTPoison require Logger + # Dialyzer has false positives for this module due to complex control flow analysis. + # The functions and patterns are reachable at runtime despite Dialyzer's pessimism. + @dialyzer {:no_match, {:generate_ai_comments, 2}} + @dialyzer {:unused_fun, {:check_llm_grading_parameters, 4}} + @dialyzer {:unused_fun, {:ensure_llm_enabled, 1}} + @dialyzer {:unused_fun, {:analyze_code, 2}} + @dialyzer {:unused_fun, {:save_comment, 4}} + @dialyzer {:unused_fun, {:save_comment, 5}} + alias Cadet.{Assessments, AIComments, Courses, LLMStats} alias CadetWeb.{AICodeAnalysisController, AICommentsHelpers} @@ -117,17 +126,6 @@ defmodule CadetWeb.AICodeAnalysisController do |> put_status(:bad_request) |> text("Invalid course ID format") - {:decrypt_error, err} -> - conn - |> put_status(:internal_server_error) - |> text("Failed to decrypt LLM API key") - - # Errors for check_llm_grading_parameters - {:parameter_error, error_msg} -> - conn - |> put_status(:bad_request) - |> text(error_msg) - {:error, {status, message}} -> conn |> put_status(status) From c81065a9e61225d5892a0aa8a52ee836a483b390 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 17:32:27 +0800 Subject: [PATCH 61/68] added has_llm_content to assessment --- lib/cadet/courses/course.ex | 7 ++++++- lib/cadet/courses/courses.ex | 4 +--- .../admin_grading_controller.ex | 5 ----- .../controllers/courses_controller.ex | 5 ----- .../controllers/generate_ai_comments.ex | 20 ++++++++++--------- lib/cadet_web/helpers/ai_comments_helpers.ex | 2 +- 6 files changed, 19 insertions(+), 24 deletions(-) diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index 7cb16b2d6..8906a7ab1 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -27,7 +27,9 @@ defmodule Cadet.Courses.Course do source_chapter: integer(), source_variant: String.t(), module_help_text: String.t(), - assets_prefix: String.t() | nil + assets_prefix: String.t() | nil, + has_llm_content: boolean(), + assessment_configs: [String.t()] } schema "courses" do @@ -57,6 +59,9 @@ defmodule Cadet.Courses.Course do # Virtual field computed at runtime based on assessments in course field(:has_llm_content, :boolean, virtual: true, default: false) + # Virtual field populated at runtime by get_course_config/1 + field(:assessment_configs, {:array, :string}, virtual: true, default: []) + has_many(:assessment_config, AssessmentConfig) timestamps() diff --git a/lib/cadet/courses/courses.ex b/lib/cadet/courses/courses.ex index f2d1c8077..da7703cfc 100644 --- a/lib/cadet/courses/courses.ex +++ b/lib/cadet/courses/courses.ex @@ -82,9 +82,7 @@ defmodule Cadet.Courses do Logger.info("Successfully retrieved course configuration for course #{course_id}") {:ok, - course - |> Map.put(:assessment_configs, assessment_configs) - |> Map.put(:has_llm_content, has_llm_content)} + %{course | assessment_configs: assessment_configs, has_llm_content: has_llm_content}} end end diff --git a/lib/cadet_web/admin_controllers/admin_grading_controller.ex b/lib/cadet_web/admin_controllers/admin_grading_controller.ex index 3f9a52e8a..564988b9f 100644 --- a/lib/cadet_web/admin_controllers/admin_grading_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_grading_controller.ex @@ -2,11 +2,6 @@ defmodule CadetWeb.AdminGradingController do use CadetWeb, :controller use PhoenixSwagger - # Dialyzer false positive: Dialyzer cannot infer that assessment.course_id is guaranteed - # to be a positive integer, so it pessimistically assumes get_course_config may only - # return error. The code is safe because it handles both cases properly. - @dialyzer {:no_match, {:show, 2}} - alias Cadet.{Assessments, Courses} @doc """ diff --git a/lib/cadet_web/controllers/courses_controller.ex b/lib/cadet_web/controllers/courses_controller.ex index 59ee24c0d..3ad4c415d 100644 --- a/lib/cadet_web/controllers/courses_controller.ex +++ b/lib/cadet_web/controllers/courses_controller.ex @@ -4,11 +4,6 @@ defmodule CadetWeb.CoursesController do use PhoenixSwagger require Logger - # Dialyzer false positive: Dialyzer cannot infer that course_id is guaranteed to be - # a positive integer, so it pessimistically assumes get_course_config may only return - # error. The code is safe because it handles both cases properly. - @dialyzer {:no_match, {:index, 2}} - alias Cadet.Courses alias Cadet.Accounts.CourseRegistrations diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index a38fe5b0d..ce8d37798 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -4,15 +4,6 @@ defmodule CadetWeb.AICodeAnalysisController do require HTTPoison require Logger - # Dialyzer has false positives for this module due to complex control flow analysis. - # The functions and patterns are reachable at runtime despite Dialyzer's pessimism. - @dialyzer {:no_match, {:generate_ai_comments, 2}} - @dialyzer {:unused_fun, {:check_llm_grading_parameters, 4}} - @dialyzer {:unused_fun, {:ensure_llm_enabled, 1}} - @dialyzer {:unused_fun, {:analyze_code, 2}} - @dialyzer {:unused_fun, {:save_comment, 4}} - @dialyzer {:unused_fun, {:save_comment, 5}} - alias Cadet.{Assessments, AIComments, Courses, LLMStats} alias CadetWeb.{AICodeAnalysisController, AICommentsHelpers} @@ -126,6 +117,17 @@ defmodule CadetWeb.AICodeAnalysisController do |> put_status(:bad_request) |> text("Invalid course ID format") + {:decrypt_error, _err} -> + conn + |> put_status(:internal_server_error) + |> text("Failed to decrypt LLM API key") + + # Errors for check_llm_grading_parameters + {:parameter_error, error_msg} -> + conn + |> put_status(:bad_request) + |> text(error_msg) + {:error, {status, message}} -> conn |> put_status(status) diff --git a/lib/cadet_web/helpers/ai_comments_helpers.ex b/lib/cadet_web/helpers/ai_comments_helpers.ex index 4e2df33c9..965e501b2 100644 --- a/lib/cadet_web/helpers/ai_comments_helpers.ex +++ b/lib/cadet_web/helpers/ai_comments_helpers.ex @@ -4,7 +4,7 @@ defmodule CadetWeb.AICommentsHelpers do """ require Logger - def decrypt_llm_api_key(nil), do: nil + def decrypt_llm_api_key(nil), do: {:decrypt_error, :no_api_key_configured} def decrypt_llm_api_key(encrypted_key) do case Application.get_env(:openai, :encryption_key) do From 0755dc3fe632fdcdb014b66a29a3eeeee2913a05 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 18:11:13 +0800 Subject: [PATCH 62/68] fixing coverage for string course ID --- test/cadet/assessments/assessments_test.exs | 27 ++++++++++--------- test/cadet/courses/courses_test.exs | 10 +++++++ .../ai_code_analysis_controller_test.exs | 24 +++++++++++++++++ 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/test/cadet/assessments/assessments_test.exs b/test/cadet/assessments/assessments_test.exs index ba7744a45..18f3c7d0c 100644 --- a/test/cadet/assessments/assessments_test.exs +++ b/test/cadet/assessments/assessments_test.exs @@ -2669,10 +2669,8 @@ defmodule Cadet.AssessmentsTest do test "filter by student username 1", %{ course_regs: %{avenger1_cr: avenger, students: students}, - assessments: assessments, total_submissions: total_submissions } do - expected_length = length(Map.keys(assessments)) student = Enum.at(students, 0) student_username = student.user.username @@ -2684,19 +2682,20 @@ defmodule Cadet.AssessmentsTest do submissions_from_res = res[:data][:submissions] - assert length(submissions_from_res) == expected_length + assert length(submissions_from_res) > 0 Enum.each(submissions_from_res, fn s -> - assert s.student_id == student.id + submission_student = Enum.find(students, fn st -> st.id == s.student_id end) + assert String.contains?(submission_student.user.username, student_username) end) + + assert Enum.any?(submissions_from_res, fn s -> s.student_id == student.id end) end test "filter by student username 2", %{ course_regs: %{avenger1_cr: avenger, students: students}, - assessments: assessments, total_submissions: total_submissions } do - expected_length = length(Map.keys(assessments)) student = Enum.at(students, 1) student_username = student.user.username @@ -2708,19 +2707,20 @@ defmodule Cadet.AssessmentsTest do submissions_from_res = res[:data][:submissions] - assert length(submissions_from_res) == expected_length + assert length(submissions_from_res) > 0 Enum.each(submissions_from_res, fn s -> - assert s.student_id == student.id + submission_student = Enum.find(students, fn st -> st.id == s.student_id end) + assert String.contains?(submission_student.user.username, student_username) end) + + assert Enum.any?(submissions_from_res, fn s -> s.student_id == student.id end) end test "filter by student username 3", %{ course_regs: %{avenger1_cr: avenger, students: students}, - assessments: assessments, total_submissions: total_submissions } do - expected_length = length(Map.keys(assessments)) student = Enum.at(students, 2) student_username = student.user.username @@ -2732,11 +2732,14 @@ defmodule Cadet.AssessmentsTest do submissions_from_res = res[:data][:submissions] - assert length(submissions_from_res) == expected_length + assert length(submissions_from_res) > 0 Enum.each(submissions_from_res, fn s -> - assert s.student_id == student.id + submission_student = Enum.find(students, fn st -> st.id == s.student_id end) + assert String.contains?(submission_student.user.username, student_username) end) + + assert Enum.any?(submissions_from_res, fn s -> s.student_id == student.id end) end test "filter by assessment config 1", %{ diff --git a/test/cadet/courses/courses_test.exs b/test/cadet/courses/courses_test.exs index 0a0e3a91c..9c2832e1b 100644 --- a/test/cadet/courses/courses_test.exs +++ b/test/cadet/courses/courses_test.exs @@ -67,6 +67,16 @@ defmodule Cadet.CoursesTest do assert course.assessment_configs == ["Missions", "Quests"] end + test "succeeds with string course id" do + course = insert(:course) + insert(:assessment_config, %{order: 1, type: "Missions", course: course}) + insert(:assessment_config, %{order: 2, type: "Quests", course: course}) + + {:ok, loaded_course} = Courses.get_course_config(Integer.to_string(course.id)) + assert loaded_course.id == course.id + assert loaded_course.assessment_configs == ["Missions", "Quests"] + end + test "returns with error for invalid course id" do course = insert(:course) diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index cb1ca9af7..654c4ba69 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -117,6 +117,30 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do assert response(conn, 403) == "Forbidden" end + test "errors out when LLM API key is missing", %{conn: conn} do + course_without_key = + insert(:course, %{ + enable_llm_grading: true, + llm_api_key: nil, + llm_model: "gpt-5-mini", + llm_api_url: "http://testapi.com", + llm_course_level_prompt: "Example Prompt" + }) + + assessment = insert(:assessment, %{course: course_without_key}) + submission = insert(:submission, %{assessment: assessment}) + question = insert(:programming_question, %{assessment: assessment}) + answer = insert(:answer, %{submission: submission, question: question}) + admin_user = insert(:course_registration, %{role: :admin, course: course_without_key}) + + conn = + conn + |> sign_in(admin_user.user) + |> post(build_url_generate_ai_comments(course_without_key.id, answer.id)) + + assert response(conn, 500) == "Failed to decrypt LLM API key" + end + test "LLM endpoint returns an invalid response - should log errors in database", %{ conn: conn, admin_user: admin_user, From ab7d85a94a617e5ffa40596ee1edcb868f81bea4 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 18:39:08 +0800 Subject: [PATCH 63/68] fixing public LLMStats functions --- .../admin_llm_stats_controller.ex | 99 ++++++++++++++----- 1 file changed, 73 insertions(+), 26 deletions(-) diff --git a/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex b/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex index 6dfb8c8a4..3477f3562 100644 --- a/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex @@ -13,13 +13,22 @@ defmodule CadetWeb.AdminLLMStatsController do Returns assessment-level LLM usage statistics with per-question breakdown. """ def course_stats(conn, %{"course_id" => course_id}) do - stats = LLMStats.get_course_statistics(course_id) - json(conn, stats) + with {:ok, course_id} <- parse_id(course_id) do + stats = LLMStats.get_course_statistics(course_id) + json(conn, stats) + else + :error -> conn |> put_status(:bad_request) |> text("Invalid course_id") + end end def assessment_stats(conn, %{"course_id" => course_id, "assessment_id" => assessment_id}) do - stats = LLMStats.get_assessment_statistics(course_id, assessment_id) - json(conn, stats) + with {:ok, course_id} <- parse_id(course_id), + {:ok, assessment_id} <- parse_id(assessment_id) do + stats = LLMStats.get_assessment_statistics(course_id, assessment_id) + json(conn, stats) + else + :error -> conn |> put_status(:bad_request) |> text("Invalid course_id or assessment_id") + end end @doc """ @@ -31,8 +40,17 @@ defmodule CadetWeb.AdminLLMStatsController do "assessment_id" => assessment_id, "question_id" => question_id }) do - stats = LLMStats.get_question_statistics(course_id, assessment_id, question_id) - json(conn, stats) + with {:ok, course_id} <- parse_id(course_id), + {:ok, assessment_id} <- parse_id(assessment_id), + {:ok, question_id} <- parse_id(question_id) do + stats = LLMStats.get_question_statistics(course_id, assessment_id, question_id) + json(conn, stats) + else + :error -> + conn + |> put_status(:bad_request) + |> text("Invalid course_id, assessment_id, or question_id") + end end @doc """ @@ -40,9 +58,17 @@ defmodule CadetWeb.AdminLLMStatsController do Returns feedback for an assessment, optionally filtered by question_id query param. """ def get_feedback(conn, params = %{"course_id" => course_id, "assessment_id" => assessment_id}) do - question_id = Map.get(params, "question_id") - feedback = LLMStats.get_feedback(course_id, assessment_id, question_id) - json(conn, feedback) + with {:ok, course_id} <- parse_id(course_id), + {:ok, assessment_id} <- parse_id(assessment_id), + {:ok, question_id} <- parse_optional_id(Map.get(params, "question_id")) do + feedback = LLMStats.get_feedback(course_id, assessment_id, question_id) + json(conn, feedback) + else + :error -> + conn + |> put_status(:bad_request) + |> text("Invalid course_id, assessment_id, or question_id") + end end @doc """ @@ -53,29 +79,50 @@ defmodule CadetWeb.AdminLLMStatsController do conn, params = %{"course_id" => course_id, "assessment_id" => assessment_id} ) do - user = conn.assigns[:current_user] + with {:ok, course_id} <- parse_id(course_id), + {:ok, assessment_id} <- parse_id(assessment_id), + {:ok, question_id} <- parse_optional_id(Map.get(params, "question_id")) do + user = conn.assigns[:current_user] - attrs = %{ - course_id: course_id, - user_id: user.id, - assessment_id: assessment_id, - question_id: Map.get(params, "question_id"), - rating: Map.get(params, "rating"), - body: Map.get(params, "body") - } + attrs = %{ + course_id: course_id, + user_id: user.id, + assessment_id: assessment_id, + question_id: question_id, + rating: Map.get(params, "rating"), + body: Map.get(params, "body") + } - case LLMStats.submit_feedback(attrs) do - {:ok, _feedback} -> - conn - |> put_status(:created) - |> json(%{message: "Feedback submitted successfully"}) + case LLMStats.submit_feedback(attrs) do + {:ok, _feedback} -> + conn + |> put_status(:created) + |> json(%{message: "Feedback submitted successfully"}) - {:error, changeset} -> - Logger.error("Failed to submit LLM feedback: #{inspect(changeset.errors)}") + {:error, changeset} -> + Logger.error("Failed to submit LLM feedback: #{inspect(changeset.errors)}") + conn + |> put_status(:bad_request) + |> json(%{error: "Failed to submit feedback"}) + end + else + :error -> conn |> put_status(:bad_request) - |> json(%{error: "Failed to submit feedback"}) + |> text("Invalid course_id, assessment_id, or question_id") end end + + defp parse_id(id) when is_integer(id), do: {:ok, id} + + defp parse_id(id) when is_binary(id) do + case Integer.parse(id) do + {parsed, ""} -> {:ok, parsed} + _ -> :error + end + end + + defp parse_optional_id(nil), do: {:ok, nil} + defp parse_optional_id(id), do: parse_id(id) end From 0e9c42925524d199a35efc8a7bc11a927bd16fb1 Mon Sep 17 00:00:00 2001 From: Leong Yi Quan Date: Sat, 28 Mar 2026 18:53:31 +0800 Subject: [PATCH 64/68] fixed readability --- .../admin_controllers/admin_llm_stats_controller.ex | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex b/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex index 3477f3562..fcf883b0b 100644 --- a/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_llm_stats_controller.ex @@ -13,11 +13,13 @@ defmodule CadetWeb.AdminLLMStatsController do Returns assessment-level LLM usage statistics with per-question breakdown. """ def course_stats(conn, %{"course_id" => course_id}) do - with {:ok, course_id} <- parse_id(course_id) do - stats = LLMStats.get_course_statistics(course_id) - json(conn, stats) - else - :error -> conn |> put_status(:bad_request) |> text("Invalid course_id") + case parse_id(course_id) do + {:ok, course_id} -> + stats = LLMStats.get_course_statistics(course_id) + json(conn, stats) + + :error -> + conn |> put_status(:bad_request) |> text("Invalid course_id") end end From 7fe996a878f782659d0437ecb7677424a8fee5f5 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Mon, 30 Mar 2026 21:40:07 +0800 Subject: [PATCH 65/68] fix(xml-parser): support LLM_QUESTION_PROMPT and legacy LLM_GRADING_PROMPT for task-level llm_prompt --- lib/cadet/jobs/xml_parser.ex | 4 +- test/cadet/updater/xml_parser_test.exs | 74 ++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/lib/cadet/jobs/xml_parser.ex b/lib/cadet/jobs/xml_parser.ex index 206ed8c60..caf575bf7 100644 --- a/lib/cadet/jobs/xml_parser.ex +++ b/lib/cadet/jobs/xml_parser.ex @@ -205,7 +205,9 @@ defmodule Cadet.Updater.XMLParser do template: ~x"./SNIPPET/TEMPLATE/text()" |> transform_by(&process_charlist/1), postpend: ~x"./SNIPPET/POSTPEND/text()" |> transform_by(&process_charlist/1), solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1), - llm_prompt: ~x"./LLM_GRADING_PROMPT/text()" |> transform_by(&process_charlist/1) + llm_prompt: + ~x"(./LLM_QUESTION_PROMPT/text() | ./LLM_GRADING_PROMPT/text())[1]"so + |> transform_by(&process_charlist/1) ), entity |> xmap( diff --git a/test/cadet/updater/xml_parser_test.exs b/test/cadet/updater/xml_parser_test.exs index 6fe4e3e1f..8846d5e98 100644 --- a/test/cadet/updater/xml_parser_test.exs +++ b/test/cadet/updater/xml_parser_test.exs @@ -275,6 +275,80 @@ defmodule Cadet.Updater.XMLParserTest do "Assessment has submissions, ignoring..." end end + + test "maps LLM_QUESTION_PROMPT to programming question llm_prompt", %{ + assessment_configs: [assessment_config | _], + course: course + } do + xml = """ + + + + None + Summary + Long summary + + + Prompted programming question + + Use rubric A. + + + + + """ + + assert :ok == XMLParser.parse_xml(xml, course.id, assessment_config.id) + + assessment = + Assessment + |> where(number: "LLM_PROMPT_Q") + |> Repo.one!() + + question = + Question + |> where(assessment_id: ^assessment.id) + |> Repo.one!() + + assert question.question["llm_prompt"] == "Use rubric A." + end + + test "keeps supporting legacy LLM_GRADING_PROMPT tag", %{ + assessment_configs: [assessment_config | _], + course: course + } do + xml = """ + + + + None + Summary + Long summary + + + Legacy prompted programming question + + Use rubric B. + + + + + """ + + assert :ok == XMLParser.parse_xml(xml, course.id, assessment_config.id) + + assessment = + Assessment + |> where(number: "LLM_PROMPT_G") + |> Repo.one!() + + question = + Question + |> where(assessment_id: ^assessment.id) + |> Repo.one!() + + assert question.question["llm_prompt"] == "Use rubric B." + end end describe "XML file processing" do From f0b6c826005a3619d4f66d8b6f60d2dc7175cb25 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Mon, 30 Mar 2026 21:40:34 +0800 Subject: [PATCH 66/68] fix(grading): require course enable plus mission and task prompts before exposing AI grading prompts --- .../admin_views/admin_grading_view.ex | 11 ++- .../admin_grading_controller_test.exs | 76 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index fd9156f24..8388a1de5 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -244,7 +244,13 @@ defmodule CadetWeb.AdminGradingView do end defp build_prompts(answer, course, assessment) do - if course.enable_llm_grading do + question_prompt = + Map.get(answer.question.question, "llm_prompt") || + Map.get(answer.question.question, :llm_prompt) + + if course.enable_llm_grading && + present_prompt?(assessment.llm_assessment_prompt) && + present_prompt?(question_prompt) do AICodeAnalysisController.create_final_messages( course.llm_course_level_prompt, assessment.llm_assessment_prompt, @@ -255,6 +261,9 @@ defmodule CadetWeb.AdminGradingView do end end + defp present_prompt?(value) when is_binary(value), do: String.trim(value) != "" + defp present_prompt?(_), do: false + defp build_grade(answer = %{grader: grader}) do transform_map_for_view(answer, %{ grader: grader_builder(grader), diff --git a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs index 6554b669b..0633b98ca 100644 --- a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs @@ -454,6 +454,82 @@ defmodule CadetWeb.AdminGradingControllerTest do conn = get(conn, build_url(course_id, 1)) assert response(conn, 400) == "Submission is not found." end + + @tag authenticate: :staff + test "returns prompts when both mission and question prompts are present", %{conn: conn} do + %{course: course, mission: mission, questions: questions, submissions: [submission | _]} = + seed_db(conn) + + course + |> Ecto.Changeset.change(enable_llm_grading: true) + |> Repo.update!() + + mission + |> Ecto.Changeset.change(llm_assessment_prompt: "Mission-level prompt") + |> Repo.update!() + + questions + |> Enum.filter(&(&1.type == :programming)) + |> Enum.each(fn programming_question -> + programming_question + |> Ecto.Changeset.change( + question: Map.put(programming_question.question, "llm_prompt", "Task-level prompt") + ) + |> Repo.update!() + end) + + res = + conn + |> get(build_url(course.id, submission.id)) + |> json_response(200) + + programming_answers = Enum.filter(res["answers"], &(&1["question"]["type"] == "programming")) + assert Enum.all?(programming_answers, &(length(&1["prompts"]) == 2)) + end + + @tag authenticate: :staff + test "returns empty prompts when mission-level prompt is missing", %{conn: conn} do + %{course: course, questions: questions, submissions: [submission | _]} = seed_db(conn) + + course + |> Ecto.Changeset.change(enable_llm_grading: true) + |> Repo.update!() + + programming_question = Enum.find(questions, &(&1.type == :programming)) + + programming_question + |> Ecto.Changeset.change(question: Map.put(programming_question.question, "llm_prompt", "Task-level prompt")) + |> Repo.update!() + + res = + conn + |> get(build_url(course.id, submission.id)) + |> json_response(200) + + programming_answer = Enum.find(res["answers"], &(&1["question"]["type"] == "programming")) + assert programming_answer["prompts"] == [] + end + + @tag authenticate: :staff + test "returns empty prompts when question-level prompt is missing", %{conn: conn} do + %{course: course, mission: mission, submissions: [submission | _]} = seed_db(conn) + + course + |> Ecto.Changeset.change(enable_llm_grading: true) + |> Repo.update!() + + mission + |> Ecto.Changeset.change(llm_assessment_prompt: "Mission-level prompt") + |> Repo.update!() + + res = + conn + |> get(build_url(course.id, submission.id)) + |> json_response(200) + + programming_answer = Enum.find(res["answers"], &(&1["question"]["type"] == "programming")) + assert programming_answer["prompts"] == [] + end end describe "POST /:submissionid/:questionid, staff" do From 70d9e03bd44a862e7531cfc06ffd915de368ff6a Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Mon, 30 Mar 2026 23:19:16 +0800 Subject: [PATCH 67/68] fix: formatting --- lib/cadet_web/admin_views/admin_grading_view.ex | 4 ++-- .../admin_controllers/admin_grading_controller_test.exs | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index 8388a1de5..6414266a7 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -248,8 +248,8 @@ defmodule CadetWeb.AdminGradingView do Map.get(answer.question.question, "llm_prompt") || Map.get(answer.question.question, :llm_prompt) - if course.enable_llm_grading && - present_prompt?(assessment.llm_assessment_prompt) && + if course.enable_llm_grading && + present_prompt?(assessment.llm_assessment_prompt) && present_prompt?(question_prompt) do AICodeAnalysisController.create_final_messages( course.llm_course_level_prompt, diff --git a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs index 0633b98ca..e3d29ecbf 100644 --- a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs @@ -483,7 +483,9 @@ defmodule CadetWeb.AdminGradingControllerTest do |> get(build_url(course.id, submission.id)) |> json_response(200) - programming_answers = Enum.filter(res["answers"], &(&1["question"]["type"] == "programming")) + programming_answers = + Enum.filter(res["answers"], &(&1["question"]["type"] == "programming")) + assert Enum.all?(programming_answers, &(length(&1["prompts"]) == 2)) end @@ -498,7 +500,9 @@ defmodule CadetWeb.AdminGradingControllerTest do programming_question = Enum.find(questions, &(&1.type == :programming)) programming_question - |> Ecto.Changeset.change(question: Map.put(programming_question.question, "llm_prompt", "Task-level prompt")) + |> Ecto.Changeset.change( + question: Map.put(programming_question.question, "llm_prompt", "Task-level prompt") + ) |> Repo.update!() res = From 121d86460776b8ec02f06db99a2c9fa53819b494 Mon Sep 17 00:00:00 2001 From: tzj04 <190485478+tzj04@users.noreply.github.com> Date: Mon, 30 Mar 2026 23:29:09 +0800 Subject: [PATCH 68/68] fix: credo alias --- .../admin_grading_controller_test.exs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs index e3d29ecbf..4bf02818e 100644 --- a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs @@ -4,6 +4,7 @@ defmodule CadetWeb.AdminGradingControllerTest do alias Cadet.Assessments.{Answer, Submission} alias Cadet.Repo alias CadetWeb.AdminGradingController + alias Ecto.Changeset import Mock @@ -461,18 +462,18 @@ defmodule CadetWeb.AdminGradingControllerTest do seed_db(conn) course - |> Ecto.Changeset.change(enable_llm_grading: true) + |> Changeset.change(enable_llm_grading: true) |> Repo.update!() mission - |> Ecto.Changeset.change(llm_assessment_prompt: "Mission-level prompt") + |> Changeset.change(llm_assessment_prompt: "Mission-level prompt") |> Repo.update!() questions |> Enum.filter(&(&1.type == :programming)) |> Enum.each(fn programming_question -> programming_question - |> Ecto.Changeset.change( + |> Changeset.change( question: Map.put(programming_question.question, "llm_prompt", "Task-level prompt") ) |> Repo.update!() @@ -494,13 +495,13 @@ defmodule CadetWeb.AdminGradingControllerTest do %{course: course, questions: questions, submissions: [submission | _]} = seed_db(conn) course - |> Ecto.Changeset.change(enable_llm_grading: true) + |> Changeset.change(enable_llm_grading: true) |> Repo.update!() programming_question = Enum.find(questions, &(&1.type == :programming)) programming_question - |> Ecto.Changeset.change( + |> Changeset.change( question: Map.put(programming_question.question, "llm_prompt", "Task-level prompt") ) |> Repo.update!() @@ -519,11 +520,11 @@ defmodule CadetWeb.AdminGradingControllerTest do %{course: course, mission: mission, submissions: [submission | _]} = seed_db(conn) course - |> Ecto.Changeset.change(enable_llm_grading: true) + |> Changeset.change(enable_llm_grading: true) |> Repo.update!() mission - |> Ecto.Changeset.change(llm_assessment_prompt: "Mission-level prompt") + |> Changeset.change(llm_assessment_prompt: "Mission-level prompt") |> Repo.update!() res =