Add AI grading statistics/feedback and refined AI comment persistence workflow

.tool-versions

@@ -0,0 +1,4 @@
+elixir 1.19.0-otp-27
+erlang 27.2.4
+local 1.19.0-otp-27
+exlixir 1.19.0-otp-27

config/cadet.exs

@@ -0,0 +1,179 @@
+# Example production configuration file
+
+import Config
+
+# See https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-runtime-configuration
+# except for cors_endpoints, load_from_system_env which are custom
+config :cadet, CadetWeb.Endpoint,
+  # See https://hexdocs.pm/corsica/Corsica.html#module-origins
+  # Remove for "*"
+  cors_endpoints: "example.com",
+  server: true,
+  # If true, expects an environment variable PORT specifying the port to listen on
+  load_from_system_env: true,
+  url: [host: "api.example.com", port: 80],
+  # You can specify the port here instead
+  # e.g http: [compress: true, port: 4000]
+  http: [compress: true],
+  # Generate using `mix phx.gen.secret`
+  secret_key_base: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+
+config :cadet, Cadet.Auth.Guardian,
+  issuer: "cadet",
+  # Generate using `mix phx.gen.secret`
+  secret_key: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+
+config :cadet, Cadet.Repo,
+  # Do not change this, only Postgres is supported
+  # (This is here because of how configuration works in Elixir.)
+  adapter: Ecto.Adapters.Postgres,
+  # The AWS Secrets Manager secret name containing the database connection details
+  rds_secret_name: "<unique-identifier>-cadet-db",
+  # Alternatively, you can include the credentials here:
+  # (comment out or remove rds_secret_name)
+  # username: "postgres",
+  # password: "postgres",
+  # database: "cadet_stg",
+  # hostname: "localhost",
+  pool_size: 10
+
+config :cadet,
+  identity_providers: %{
+    # # To use authentication with ADFS.
+    # "nusnet" =>
+    #   {Cadet.Auth.Providers.ADFS,
+    #    %{
+    #      # The OAuth2 token endpoint.
+    #      token_endpoint: "https://my-adfs/adfs/oauth2/token"
+    #    }},
+    # # An example of OpenID authentication with Cognito. Any OpenID-compliant
+    # # provider should work.
+    # "cognito" =>
+    #   {Cadet.Auth.Providers.OpenID,
+    #    %{
+    #      # This should match a key in openid_connect_providers below
+    #      openid_provider: :cognito,
+    #      # You may need to write your own claim extractor for other providers
+    #      claim_extractor: Cadet.Auth.Providers.CognitoClaimExtractor
+    #    }},
+
+    # # Example SAML authentication with NUS Student IdP
+    # "test_saml" =>
+    #   {Cadet.Auth.Providers.SAML,
+    #    %{
+    #      assertion_extractor: Cadet.Auth.Providers.NusstuAssertionExtractor,
+    #      client_redirect_url: "http://cadet.frontend:8000/login/callback"
+    #    }},
+
+    "test" =>
+      {Cadet.Auth.Providers.Config,
+       [
+         %{
+           token: "admin_token",
+           code: "admin_code",
+           name: "Test Admin",
+           username: "admin",
+           role: :admin
+         },
+         %{
+           token: "staff_token",
+           code: "staff_code",
+           name: "Test Staff",
+           username: "staff",
+           role: :staff
+         },
+         %{
+           token: "student_token",
+           code: "student_code",
+           name: "Test Student",
+           username: "student",
+           role: :student
+         }
+       ]}
+  },
+  # See https://hexdocs.pm/openid_connect/readme.html
+  # openid_connect_providers: [
+  #   cognito: [
+  #     discovery_document_uri: "",
+  #     client_id: "",
+  #     client_secret: "",
+  #     response_type: "code",
+  #     scope: "openid profile"
+  #   ]
+  # ],
+  autograder: [
+    lambda_name: "<unique-identifier>-grader"
+  ],
+  uploader: [
+    assets_bucket: "<unique-identifier>-assets",
+    assets_prefix: "courses/",
+    sourcecasts_bucket: "<unique-identifier>-cadet-sourcecasts"
+  ],
+  # Configuration for Sling integration (executing on remote devices)
+  remote_execution: [
+    # Prefix for AWS IoT thing names
+    thing_prefix: "<unique-identifier>-sling",
+    # AWS IoT thing group to put created things into (must be set-up beforehand)
+    thing_group: "<unique-identifier>-sling",
+    # Role ARN to use when generating signed client MQTT-WS URLs (must be set-up beforehand)
+    # Note, you need to specify the correct account ID. Find it in AWS IAM at the bottom left.
+    client_role_arn: "arn:aws:iam::<account-id>:role/<unique-identifier>-cadet-frontend"
+  ]
+
+# Sentry DSN. This is only really useful to the NUS SoC deployments, but you can
+# specify a DSN here if you wish to track backend errors.
+# config :sentry,
+#   dsn: "https://public_key/sentry.io/somethingsomething"
+
+# If you are not running on EC2, you will need to configure an AWS access token
+# for the backend to access AWS resources:
+#
+# This will make ExAWS read the values from the corresponding environment variables.
+# config :ex_aws,
+#   access_key_id: [{:system, "AWS_ACCESS_KEY_ID"}],
+#   secret_access_key: [{:system, "AWS_SECRET_ACCESS_KEY"}]
+#
+# You can also just specify the values directly:
+# config :ex_aws,
+#   access_key_id: "ACCESS KEY",
+#   secret_access_key: "SECRET KEY"
+#
+# You may also want to change the AWS or S3 region used for all resources:
+# (Note, the default is ap-southeast-1 i.e. Singapore)
+# config :ex_aws,
+#   region: "ap-southeast-1",
+#   s3: [
+#     scheme: "https://",
+#     host: "s3.ap-southeast-1.amazonaws.com",
+#     region: "ap-southeast-1"
+#   ]
+
+# You may also want to change the timezone used for scheduled jobs
+# config :cadet, Cadet.Jobs.Scheduler,
+#   timezone: "Asia/Singapore",
+
+# # Additional configuration for SAML authentication
+# # For more details, see https://github.com/handnot2/samly
+# config :samly, Samly.Provider,
+#   idp_id_from: :path_segment,
+#   service_providers: [
+#     %{
+#       id: "source-academy-backend",
+#       certfile: "example_path/certfile.pem",
+#       keyfile: "example_path/keyfile.pem"
+#     }
+#   ],
+#   identity_providers: [
+#     %{
+#       id: "student",
+#       sp_id: "source-academy-backend",
+#       base_url: "https://example_backend/sso",
+#       metadata_file: "student_idp_metadata.xml"
+#     },
+#     %{
+#       id: "staff",
+#       sp_id: "source-academy-backend",
+#       base_url: "https://example_backend/sso",
+#       metadata_file: "staff_idp_metadata.xml"
+#     }
+#   ]

lib/cadet/ai_comments.ex

@@ -5,7 +5,7 @@ defmodule Cadet.AIComments do
 
   import Ecto.Query
   alias Cadet.Repo
-  alias Cadet.AIComments.AIComment
+  alias Cadet.AIComments.{AIComment, AICommentVersion}
 
   @doc """
   Creates a new AI comment log entry.
@@ -41,37 +41,87 @@ defmodule Cadet.AIComments do
   end
 
   @doc """
-  Updates the final comment for a specific submission and question.
-  Returns the most recent comment entry for that submission/question.
+  Updates an existing AI comment with new attributes.
   """
-  def update_final_comment(answer_id, final_comment) do
-    comment = get_latest_ai_comment(answer_id)
-
-    case comment do
-      nil ->
+  def update_ai_comment(id, attrs) do
+    id
+    |> get_ai_comment()
+    |> case do
+      {:error, :not_found} ->
         {:error, :not_found}
 
-      _ ->
+      {:ok, comment} ->
         comment
-        |> AIComment.changeset(%{final_comment: final_comment})
+        |> AIComment.changeset(attrs)
         |> Repo.update()
     end
   end
 
   @doc """
-  Updates an existing AI comment with new attributes.
+  Saves selected comment indices and finalization metadata for an AI comment.
   """
-  def update_ai_comment(id, attrs) do
-    id
-    |> get_ai_comment()
-    |> case do
-      {:error, :not_found} ->
+  def save_selected_comments(answer_id, selected_indices, finalized_by_id) do
+    case get_latest_ai_comment(answer_id) do
+      nil ->
         {:error, :not_found}
 
-      {:ok, comment} ->
+      comment ->
         comment
-        |> AIComment.changeset(attrs)
+        |> AIComment.changeset(%{
+          selected_indices: selected_indices,
+          finalized_by_id: finalized_by_id,
+          finalized_at: DateTime.truncate(DateTime.utc_now(), :second)
+        })
         |> Repo.update()
     end
   end
+
+  @doc """
+  Creates a new version entry for a specific comment index.
+  Automatically determines the next version number.
+  """
+  def create_comment_version(ai_comment_id, comment_index, content, editor_id) do
+    next_version =
+      Repo.one(
+        from(v in AICommentVersion,
+          where: v.ai_comment_id == ^ai_comment_id and v.comment_index == ^comment_index,
+          select: coalesce(max(v.version_number), 0)
+        )
+      ) + 1
+
+    %AICommentVersion{}
+    |> AICommentVersion.changeset(%{
+      ai_comment_id: ai_comment_id,
+      comment_index: comment_index,
+      version_number: next_version,
+      content: content,
+      editor_id: editor_id
+    })
+    |> Repo.insert()
+  end
+
+  @doc """
+  Gets all versions for a specific AI comment, ordered by comment_index and version_number.
+  """
+  def get_comment_versions(ai_comment_id) do
+    Repo.all(
+      from(v in AICommentVersion,
+        where: v.ai_comment_id == ^ai_comment_id,
+        order_by: [asc: v.comment_index, asc: v.version_number]
+      )
+    )
+  end
+
+  @doc """
+  Gets the latest version for a specific comment index.
+  """
+  def get_latest_version(ai_comment_id, comment_index) do
+    Repo.one(
+      from(v in AICommentVersion,
+        where: v.ai_comment_id == ^ai_comment_id and v.comment_index == ^comment_index,
+        order_by: [desc: v.version_number],
+        limit: 1
+      )
+    )
+  end
 end

lib/cadet/ai_comments/ai_comment.ex

@@ -11,15 +11,19 @@ defmodule Cadet.AIComments.AIComment do
     field(:answers_json, :string)
     field(:response, :string)
     field(:error, :string)
-    field(:final_comment, :string)
+    field(:selected_indices, {:array, :integer})
+    field(:finalized_at, :utc_datetime)
 
     belongs_to(:answer, Cadet.Assessments.Answer)
+    belongs_to(:finalized_by, Cadet.Accounts.User, foreign_key: :finalized_by_id)
+
+    has_many(:versions, Cadet.AIComments.AICommentVersion)
 
     timestamps()
   end
 
   @required_fields ~w(answer_id raw_prompt answers_json)a
-  @optional_fields ~w(response error final_comment)a
+  @optional_fields ~w(response error selected_indices finalized_by_id finalized_at)a
 
   def changeset(ai_comment, attrs) do
     ai_comment

lib/cadet/ai_comments/ai_comment_version.ex

@@ -0,0 +1,32 @@
+defmodule Cadet.AIComments.AICommentVersion do
+  @moduledoc """
+  Defines the schema and changeset for AI comment versions.
+  Tracks per-comment edits made by tutors.
+  """
+
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  schema "ai_comment_versions" do
+    field(:comment_index, :integer)
+    field(:version_number, :integer)
+    field(:content, :string)
+
+    belongs_to(:ai_comment, Cadet.AIComments.AIComment)
+    belongs_to(:editor, Cadet.Accounts.User, foreign_key: :editor_id)
+
+    timestamps()
+  end
+
+  @required_fields ~w(ai_comment_id comment_index version_number content)a
+  @optional_fields ~w(editor_id)a
+
+  def changeset(version, attrs) do
+    version
+    |> cast(attrs, @required_fields ++ @optional_fields)
+    |> validate_required(@required_fields)
+    |> foreign_key_constraint(:ai_comment_id)
+    |> foreign_key_constraint(:editor_id)
+    |> unique_constraint([:ai_comment_id, :comment_index, :version_number])
+  end
+end

lib/cadet/assessments/assessment.ex

@@ -21,6 +21,7 @@ defmodule Cadet.Assessments.Assessment do
     field(:question_count, :integer, virtual: true)
     field(:graded_count, :integer, virtual: true)
     field(:is_grading_published, :boolean, virtual: true)
+    field(:has_llm_questions, :boolean, virtual: true, default: false)
     field(:title, :string)
     field(:is_published, :boolean, default: false)
     field(:summary_short, :string)

lib/cadet/assessments/query.ex

@@ -20,7 +20,8 @@ defmodule Cadet.Assessments.Query do
     |> select([a, q], %Assessment{
       a
       | max_xp: q.max_xp,
-        question_count: q.question_count
+        question_count: q.question_count,
+        has_llm_questions: q.has_llm_questions
     })
   end
 
@@ -49,7 +50,9 @@ defmodule Cadet.Assessments.Query do
     |> select([q], %{
       assessment_id: q.assessment_id,
       max_xp: sum(q.max_xp),
-      question_count: count(q.id)
+      question_count: count(q.id),
+      has_llm_questions:
+        fragment("bool_or(? ->> 'llm_prompt' IS NOT NULL AND ? ->> 'llm_prompt' != '')", q.question, q.question)
     })
   end
 end