Add AI grading statistics/feedback and refined AI comment persistence workflow

.gitignore

@@ -94,6 +94,8 @@ erl_crash.dump
 # variables.
 /config/*secrets.exs
 
+/config/cadet.exs
+
 # Uploaded file
 /cs1101s
 /uploads

config/config.exs

@@ -63,10 +63,10 @@ config :ex_aws,
 
 config :ex_aws, :hackney_opts, recv_timeout: 660_000
 
-# Configure Arc File Upload
-config :arc, virtual_host: true
+# Configure Waffle File Upload
+config :waffle, virtual_host: true
 # Or uncomment below to use local storage
-# config :arc, storage: Arc.Storage.Local
+# config :waffle, storage: Waffle.Storage.Local
 
 # Configures Sentry
 config :sentry,

config/runtime.exs

@@ -0,0 +1,9 @@
+import Config
+
+# This file is executed after the code compilation on all environments.
+# It contains runtime configuration that's evaluated when the system starts.
+
+# Configure the port from environment variable if set
+if port = System.get_env("PORT") do
+  config :cadet, CadetWeb.Endpoint, http: [:inet6, port: String.to_integer(port)]
+end

config/test.exs

@@ -90,7 +90,7 @@ config :cadet,
     client_role_arn: "test"
   ]
 
-config :arc, storage: Arc.Storage.Local
+config :waffle, storage: Waffle.Storage.Local
 
 if "test.secrets.exs" |> Path.expand(__DIR__) |> File.exists?(),
   do: import_config("test.secrets.exs")

lib/cadet.ex

@@ -40,8 +40,8 @@ defmodule Cadet do
 
   def remote_assets do
     quote do
-      use Arc.Definition
-      use Arc.Ecto.Definition
+      use Waffle.Definition
+      use Waffle.Ecto.Definition
     end
   end
 end

lib/cadet/ai_comments.ex

@@ -5,7 +5,7 @@ defmodule Cadet.AIComments do
 
   import Ecto.Query
   alias Cadet.Repo
-  alias Cadet.AIComments.AIComment
+  alias Cadet.AIComments.{AIComment, AICommentVersion}
 
   @doc """
   Creates a new AI comment log entry.
@@ -41,37 +41,106 @@ defmodule Cadet.AIComments do
   end
 
   @doc """
-  Updates the final comment for a specific submission and question.
-  Returns the most recent comment entry for that submission/question.
+  Updates an existing AI comment with new attributes.
   """
-  def update_final_comment(answer_id, final_comment) do
-    comment = get_latest_ai_comment(answer_id)
-
-    case comment do
-      nil ->
+  def update_ai_comment(id, attrs) do
+    id
+    |> get_ai_comment()
+    |> case do
+      {:error, :not_found} ->
         {:error, :not_found}
 
-      _ ->
+      {:ok, comment} ->
         comment
-        |> AIComment.changeset(%{final_comment: final_comment})
+        |> AIComment.changeset(attrs)
         |> Repo.update()
     end
   end
 
   @doc """
-  Updates an existing AI comment with new attributes.
+  Saves selected comment indices and finalization metadata for an AI comment.
   """
-  def update_ai_comment(id, attrs) do
-    id
-    |> get_ai_comment()
-    |> case do
-      {:error, :not_found} ->
+  def save_selected_comments(answer_id, selected_indices, finalized_by_id) do
+    case get_latest_ai_comment(answer_id) do
+      nil ->
         {:error, :not_found}
 
-      {:ok, comment} ->
+      comment ->
         comment
-        |> AIComment.changeset(attrs)
+        |> AIComment.changeset(%{
+          selected_indices: selected_indices,
+          finalized_by_id: finalized_by_id,
+          finalized_at: DateTime.truncate(DateTime.utc_now(), :second)
+        })
         |> Repo.update()
     end
   end
+
+  @doc """
+  Creates a new version entry for a specific comment index.
+  Automatically determines the next version number.
+  """
+  def create_comment_version(ai_comment_id, comment_index, content, editor_id) do
+    transaction_result =
+      Repo.transaction(fn ->
+        # Serialize version creation per (ai_comment_id, comment_index)
+        # to avoid duplicate version numbers.
+        case Repo.query("SELECT pg_advisory_xact_lock($1, $2)", [ai_comment_id, comment_index]) do
+          {:ok, _} ->
+            next_version =
+              Repo.one(
+                from(v in AICommentVersion,
+                  where: v.ai_comment_id == ^ai_comment_id and v.comment_index == ^comment_index,
+                  select: coalesce(max(v.version_number), 0)
+                )
+              ) + 1
+
+            case %AICommentVersion{}
+                 |> AICommentVersion.changeset(%{
+                   ai_comment_id: ai_comment_id,
+                   comment_index: comment_index,
+                   version_number: next_version,
+                   content: content,
+                   editor_id: editor_id
+                 })
+                 |> Repo.insert() do
+              {:ok, version} -> version
+              {:error, changeset} -> Repo.rollback(changeset)
+            end
+
+          {:error, error} ->
+            Repo.rollback(error)
+        end
+      end)
+
+    case transaction_result do
+      {:ok, version} -> {:ok, version}
+      {:error, reason} -> {:error, reason}
+    end
+  end
+
+  @doc """
+  Gets all versions for a specific AI comment, ordered by comment_index and version_number.
+  """
+  def get_comment_versions(ai_comment_id) do
+    Repo.all(
+      from(v in AICommentVersion,
+        where: v.ai_comment_id == ^ai_comment_id,
+        order_by: [asc: v.comment_index, asc: v.version_number]
+      )
+    )
+  end
+
+  @doc """
+  Gets the latest version for a specific comment index.
+  """
+  def get_latest_version(ai_comment_id, comment_index) do
+    Repo.one(
+      from(v in AICommentVersion,
+        where: v.ai_comment_id == ^ai_comment_id and v.comment_index == ^comment_index,
+        order_by: [desc: v.version_number],
+        limit: 1
+      )
+    )
+  end
 end

lib/cadet/ai_comments/ai_comment.ex

@@ -11,15 +11,19 @@ defmodule Cadet.AIComments.AIComment do
     field(:answers_json, :string)
     field(:response, :string)
     field(:error, :string)
-    field(:final_comment, :string)
+    field(:selected_indices, {:array, :integer})
+    field(:finalized_at, :utc_datetime)
 
     belongs_to(:answer, Cadet.Assessments.Answer)
+    belongs_to(:finalized_by, Cadet.Accounts.User, foreign_key: :finalized_by_id)
+
+    has_many(:versions, Cadet.AIComments.AICommentVersion)
 
     timestamps()
   end
 
   @required_fields ~w(answer_id raw_prompt answers_json)a
-  @optional_fields ~w(response error final_comment)a
+  @optional_fields ~w(response error selected_indices finalized_by_id finalized_at)a
 
   def changeset(ai_comment, attrs) do
     ai_comment

lib/cadet/ai_comments/ai_comment_version.ex

@@ -0,0 +1,32 @@
+defmodule Cadet.AIComments.AICommentVersion do
+  @moduledoc """
+  Defines the schema and changeset for AI comment versions.
+  Tracks per-comment edits made by tutors.
+  """
+
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  schema "ai_comment_versions" do
+    field(:comment_index, :integer)
+    field(:version_number, :integer)
+    field(:content, :string)
+
+    belongs_to(:ai_comment, Cadet.AIComments.AIComment)
+    belongs_to(:editor, Cadet.Accounts.User, foreign_key: :editor_id)
+
+    timestamps()
+  end
+
+  @required_fields ~w(ai_comment_id comment_index version_number content)a
+  @optional_fields ~w(editor_id)a
+
+  def changeset(version, attrs) do
+    version
+    |> cast(attrs, @required_fields ++ @optional_fields)
+    |> validate_required(@required_fields)
+    |> foreign_key_constraint(:ai_comment_id)
+    |> foreign_key_constraint(:editor_id)
+    |> unique_constraint([:ai_comment_id, :comment_index, :version_number])
+  end
+end

lib/cadet/assessments/assessment.ex

@@ -4,7 +4,7 @@ defmodule Cadet.Assessments.Assessment do
   (mission, sidequest, path, and contest)
   """
   use Cadet, :model
-  use Arc.Ecto.Schema
+  use Waffle.Ecto.Schema
 
   alias Cadet.Repo
   alias Cadet.Assessments.{AssessmentAccess, Question, SubmissionStatus, Upload}
@@ -21,6 +21,7 @@ defmodule Cadet.Assessments.Assessment do
     field(:question_count, :integer, virtual: true)
     field(:graded_count, :integer, virtual: true)
     field(:is_grading_published, :boolean, virtual: true)
+    field(:has_llm_questions, :boolean, virtual: true, default: false)
     field(:title, :string)
     field(:is_published, :boolean, default: false)
     field(:summary_short, :string)
@@ -37,6 +38,12 @@ defmodule Cadet.Assessments.Assessment do
     field(:has_token_counter, :boolean, default: false)
     field(:has_voting_features, :boolean, default: false)
     field(:llm_assessment_prompt, :string, default: nil)
+    field(:llm_input_cost, :decimal)
+    field(:llm_output_cost, :decimal)
+    field(:llm_total_input_tokens, :integer, default: 0)
+    field(:llm_total_output_tokens, :integer, default: 0)
+    field(:llm_total_cached_tokens, :integer, default: 0)
+    field(:llm_total_cost, :decimal, default: Decimal.new("0.0"))
 
     belongs_to(:config, AssessmentConfig)
     belongs_to(:course, Course)
@@ -47,7 +54,10 @@ defmodule Cadet.Assessments.Assessment do
 
   @required_fields ~w(title open_at close_at number course_id config_id max_team_size)a
   @optional_fields ~w(reading summary_short summary_long
-    is_published story cover_picture access password has_token_counter has_voting_features llm_assessment_prompt)a
+    is_published story cover_picture access password has_token_counter
+    has_voting_features llm_assessment_prompt
+    llm_input_cost llm_output_cost llm_total_input_tokens
+    llm_total_output_tokens llm_total_cached_tokens llm_total_cost)a
   @optional_file_fields ~w(mission_pdf)a
 
   def changeset(assessment, params) do
@@ -60,12 +70,14 @@ defmodule Cadet.Assessments.Assessment do
     |> cast_attachments(params, @optional_file_fields)
     |> cast(params, @required_fields ++ @optional_fields)
     |> validate_required(@required_fields)
+    # ADD THIS LINE HERE to apply your defaults
+    |> put_default_costs()
     |> add_belongs_to_id_from_model([:config, :course], params)
     |> foreign_key_constraint(:config_id)
     |> foreign_key_constraint(:course_id)
     |> unique_constraint([:number, :course_id])
-    |> validate_config_course
-    |> validate_open_close_date
+    |> validate_config_course()
+    |> validate_open_close_date()
     |> validate_number(:max_team_size, greater_than_or_equal_to: 1)
   end
 
@@ -86,6 +98,20 @@ defmodule Cadet.Assessments.Assessment do
     end
   end
 
+  defp put_default_costs(changeset) do
+    changeset
+    |> put_fallback(:llm_input_cost, Decimal.new("3.20"))
+    |> put_fallback(:llm_output_cost, Decimal.new("12.80"))
+  end
+
+  defp put_fallback(changeset, field, default_val) do
+    if get_field(changeset, field) == nil do
+      put_change(changeset, field, default_val)
+    else
+      changeset
+    end
+  end
+
   defp validate_open_close_date(changeset) do
     validate_change(changeset, :open_at, fn :open_at, open_at ->
       if Timex.before?(open_at, get_field(changeset, :close_at)) do