1.9.0: Scenarios — rename Catalogs, edit in the dashboard, generate from a project's browser, export and import

actionagent/app/controllers/action_agent/api/explorations_controller.rb

@@ -73,7 +73,13 @@ def update_candidate
       # :replace_scenarios is asked about the evaluation the candidates
       # merge into, or about the exploration when a project's accept will
       # create that evaluation.
+      #
+      # With scenario_set_id, the candidates become scenarios of that set of
+      # one of the owner's scenario catalogs instead (see
+      # Exploration#accept_into_set!), and :replace_scenarios is asked about
+      # its catalog.
       def accept
+        return accept_into_set if params[:scenario_set_id].present?
         return unless authorize_action!(:replace_scenarios, @exploration.target_evaluation || @exploration)
 
         merged = @exploration.accept!(params[:candidate_ids], edits: accept_edits)
@@ -98,6 +104,24 @@ def stop
 
       private
 
+      def accept_into_set
+        products = ScenarioProduct.where(scenario_catalog_id: owned(ScenarioCatalog).select(:id))
+        set = ScenarioSet.where(scenario_product_id: products.select(:id)).find(params[:scenario_set_id].to_s)
+        return unless authorize_action!(:replace_scenarios, set.catalog)
+
+        merged = @exploration.accept_into_set!(params[:candidate_ids], set: set, edits: accept_edits)
+        render json: detail(@exploration.reload).merge(
+          accepted: merged.slice(:added, :updated, :unchanged),
+          scenario_set: scenario_set_summary(set)
+        )
+      end
+
+      def scenario_set_summary(set)
+        catalog = set.catalog
+        { id: set.id, key: set.key, name: set.name, product_key: set.product.key,
+          catalog: { id: catalog.id, key: catalog.key, name: catalog.name } }
+      end
+
       def set_exploration
         @exploration = owned(Exploration).find(params[:id])
       end

actionagent/app/controllers/action_agent/api/scenario_catalog_entries_controller.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  module Api
+    # Editing a scenario catalog in the dashboard, one entry at a time: its
+    # products, their sets, and the sets' scenarios. Each change is checked
+    # as the YAML importer checks a document (the records' validations, then
+    # the catalog's document rebuilt and parsed), rewrites the catalog's
+    # document and digest, and writes it to Active Storage when the host has
+    # it, so an edit and an import leave the same thing behind.
+    #
+    # Every change needs :replace_scenarios on the catalog, as an import does.
+    # A key left blank is made from the name (or a scenario's prompt).
+    class ScenarioCatalogEntriesController < BaseController
+      before_action :require_owner!
+      before_action :set_catalog
+      before_action -> { authorize_action!(:replace_scenarios, @catalog) }
+
+      rescue_from ActiveRecord::RecordInvalid do |error|
+        render json: { errors: error.record.errors.full_messages }, status: :unprocessable_entity
+      end
+      rescue_from ActiveAgent::Evals::Catalog::InvalidDocument do |error|
+        render json: { errors: [ error.message ] }, status: :unprocessable_entity
+      end
+
+      # POST /api/scenario_catalogs/:id/products
+      # { product: { key, name, description, agent_id, agent_name, project_id } }
+      def create_product
+        attributes = product_attributes
+        attributes[:key] = ScenarioCatalog.key_for(attributes[:name], @catalog.products.pluck(:key)) if attributes[:key].blank?
+        attributes[:name] = attributes[:key] if attributes[:name].blank?
+        product = change { @catalog.products.create!(attributes.merge(position: next_position(@catalog.products))) }
+        render json: { product: product.summary, catalog: full }, status: :created
+      end
+
+      # PATCH /api/scenario_catalogs/:id/products/:product_id
+      def update_product
+        product = find_product
+        change { product.update!(product_attributes.compact_blank.merge(product_links)) }
+        render json: { product: product.reload.summary, catalog: full }
+      end
+
+      # DELETE /api/scenario_catalogs/:id/products/:product_id
+      # The evaluations its sets became stay, with their runs.
+      def destroy_product
+        product = find_product
+        change { product.destroy! }
+        render json: { catalog: full }
+      end
+
+      # POST /api/scenario_catalogs/:id/products/:product_id/sets
+      # { set: { key, name, description, judge: { kind, model } } }
+      def create_set
+        product = find_product
+        attributes = set_attributes
+        attributes[:key] = ScenarioCatalog.key_for(attributes[:name], product.sets.pluck(:key)) if attributes[:key].blank?
+        attributes[:name] = attributes[:key] if attributes[:name].blank?
+        set = change { product.sets.create!(attributes.merge(position: next_position(product.sets))) }
+        render json: { set: set.summary, catalog: full }, status: :created
+      end
+
+      # PATCH /api/scenario_catalogs/:id/sets/:set_id
+      def update_set
+        set = find_set
+        change { set.update!(set_attributes.compact_blank) }
+        render json: { set: set.reload.summary, catalog: full }
+      end
+
+      # DELETE /api/scenario_catalogs/:id/sets/:set_id
+      def destroy_set
+        set = find_set
+        change { set.destroy! }
+        render json: { catalog: full }
+      end
+
+      # POST /api/scenario_catalogs/:id/sets/:set_id/scenarios
+      # { scenario: { key, prompt, notes, expect: { tools, contains, not_contains }, tags, production_only, enabled } }
+      def create_scenario
+        set = find_set
+        attributes = scenario_attributes
+        attributes[:key] = ScenarioCatalog.key_for(attributes[:prompt].to_s.split.first(6).join(" "), set.scenarios.pluck(:key)) if attributes[:key].blank?
+        if @catalog.scenario_count >= ScenarioCatalog::MAX_SCENARIOS
+          return render json: { errors: [ "A catalog holds at most #{ScenarioCatalog::MAX_SCENARIOS} scenarios" ] }, status: :unprocessable_entity
+        end
+
+        scenario = change { set.scenarios.create!(attributes.merge(position: next_position(set.scenarios))) }
+        render json: { scenario: scenario.summary, catalog: full }, status: :created
+      end
+
+      # PATCH /api/scenario_catalogs/:id/sets/:set_id/scenarios/:scenario_id
+      def update_scenario
+        scenario = find_set.scenarios.find(params[:scenario_id])
+        change { scenario.update!(scenario_attributes.except(*blank_identity(scenario_attributes))) }
+        render json: { scenario: scenario.reload.summary, catalog: full }
+      end
+
+      # DELETE /api/scenario_catalogs/:id/sets/:set_id/scenarios/:scenario_id
+      def destroy_scenario
+        scenario = find_set.scenarios.find(params[:scenario_id])
+        change { scenario.destroy! }
+        render json: { catalog: full }
+      end
+
+      private
+
+      def set_catalog
+        @catalog = owned(ScenarioCatalog).find(params[:id])
+      end
+
+      def find_product
+        @catalog.products.find(params[:product_id])
+      end
+
+      def find_set
+        ScenarioSet.where(scenario_product_id: @catalog.products.select(:id)).find(params[:set_id])
+      end
+
+      # Runs +block+ and rebuilds the catalog's document in one transaction,
+      # so a change the document cannot hold is rolled back; then writes the
+      # document to storage. Returns the block's value.
+      def change
+        result = ScenarioCatalog.transaction do
+          value = yield
+          @catalog.reload.refresh_document!
+          value
+        end
+        @catalog.sync_to_storage!
+        result
+      end
+
+      def full
+        ScenarioCatalogSerializer.full(@catalog.reload)
+      end
+
+      def next_position(relation)
+        (relation.maximum(:position) || -1) + 1
+      end
+
+      def product_attributes
+        params.require(:product).permit(:key, :name, :description, :agent_name).to_h.symbolize_keys.merge(product_links)
+      end
+
+      # The agent and project a product names by id, among the owner's own.
+      # An empty id clears it.
+      def product_links
+        source = params[:product] || {}
+        links = {}
+        if source.key?(:agent_id)
+          links[:agent] = source[:agent_id].present? ? owner_agents.find(source[:agent_id]) : nil
+        end
+        if source.key?(:project_id)
+          links[:project] = source[:project_id].present? ? owned(Project).find(source[:project_id]) : nil
+        end
+        links
+      end
+
+      def set_attributes
+        source = params.require(:set)
+        attributes = source.permit(:key, :name, :description).to_h.symbolize_keys
+        if source.key?(:judge)
+          judge = source[:judge].respond_to?(:permit) ? source[:judge].permit(:kind, :model).to_h.compact_blank : {}
+          attributes[:judge] = judge
+        end
+        attributes
+      end
+
+      def scenario_attributes
+        source = params.require(:scenario)
+        attributes = source.permit(:key, :prompt, :notes, :production_only, :enabled).to_h.symbolize_keys
+        expect = source[:expect] || source[:expectations]
+        if expect.respond_to?(:permit)
+          attributes[:expectations] = Exploration::EXPECTATION_KEYS.each_with_object({}) do |name, hash|
+            values = Array(expect[name]).flat_map { |value| value.to_s.split(",") }.map(&:strip).reject(&:blank?)
+            hash[name] = values if values.any?
+          end
+        end
+        attributes[:tags] = Array(source[:tags]).flat_map { |tag| tag.to_s.split(",") }.map(&:strip).reject(&:blank?) if source.key?(:tags)
+        attributes
+      end
+
+      # An update leaves the key and prompt alone when they arrive blank,
+      # rather than failing presence.
+      def blank_identity(attributes)
+        %i[key prompt].select { |name| attributes.key?(name) && attributes[name].blank? }
+      end
+    end
+  end
+end

actionagent/app/controllers/action_agent/api/scenario_catalogs_controller.rb

@@ -37,16 +37,20 @@ def show
 
       # POST /api/scenario_catalogs
       # One of:
+      #   catalog:    { key, name, description } for an empty catalog, built
+      #               up in the dashboard (see ScenarioCatalogEntriesController)
       #   document:   the catalog's YAML text
       #   file:       an uploaded .yml
       #   repository: owner/name of a connected repository, with ref (its
       #               default branch when omitted) and path (a directory of
       #               .yml files, .activeagents/evals by default, or one file)
-      # Answers the imported catalogs. A repository the GitHub connection
-      # has not selected needs :manage_github, as reading one for a project
-      # does.
+      # mode is "replace" (the default: the document is the catalog) or
+      # "merge" (what the document leaves out is kept). Answers the imported
+      # catalogs. A repository the GitHub connection has not selected needs
+      # :manage_github, as reading one for a project does.
       def create
         return unless authorize_action!(:replace_scenarios, ScenarioCatalog.new)
+        return create_empty if params[:document].blank? && params[:file].blank? && params[:repository].blank? && params[:catalog].present?
 
         catalogs =
           if params[:repository].present?
@@ -86,10 +90,33 @@ def destroy
         head :no_content
       end
 
-      # GET /api/scenario_catalogs/:id/export
-      # The canonical YAML, as a download.
+      # POST /api/scenario_catalogs/preview
+      # What POST /api/scenario_catalogs would change, with the same
+      # document, file or repository and mode, without writing: one
+      # ScenarioCatalogImport#preview per catalog.
+      def preview
+        previews =
+          if params[:repository].present?
+            preview_repository or return
+          else
+            [ build_import(document_param, name: params[:name].presence).preview ]
+          end
+
+        render json: { previews: previews }
+      end
+
+      # GET /api/scenario_catalogs/:id/export?product_ids=&set_ids=&scenario_ids=
+      # The canonical YAML, as a download: the whole catalog, or only the
+      # products, sets or scenarios named (comma-separated ids or arrays).
+      # disposition=inline answers it as text, for copying.
       def export
-        send_data @catalog.export_yaml, filename: "#{@catalog.key.tr('/', '-')}.yml", type: "application/x-yaml", disposition: "attachment"
+        yaml = @catalog.export_yaml(product_ids: params[:product_ids], set_ids: params[:set_ids], scenario_ids: params[:scenario_ids])
+        if params[:disposition].to_s == "inline"
+          render plain: yaml, content_type: "application/x-yaml"
+        else
+          suffix = [ params[:product_ids], params[:set_ids], params[:scenario_ids] ].any?(&:present?) ? "-selection" : ""
+          send_data yaml, filename: "#{@catalog.key.tr('/', '-')}#{suffix}.yml", type: "application/x-yaml", disposition: "attachment"
+        end
       end
 
       # POST /api/scenario_catalogs/:id/sync
@@ -180,11 +207,35 @@ def document_param
       end
 
       def import_document(document, catalog: nil, name: nil, source_path: nil)
+        build_import(document, catalog: catalog, name: name, source_path: source_path).call
+      end
+
+      def build_import(document, catalog: nil, name: nil, source_path: nil)
         ScenarioCatalogImport.new(
           owner: current_owner, document: document, catalog: catalog, name: name,
           source_kind: "upload", source_path: source_path,
-          agents: owner_agents, projects: owned(Project)
-        ).call
+          agents: owner_agents, projects: owned(Project), mode: params[:mode].presence || "replace"
+        )
+      end
+
+      # An empty catalog from catalog: { key, name, description }; a blank
+      # key is made from the name.
+      def create_empty
+        attributes = params.require(:catalog).permit(:key, :name, :description).to_h.symbolize_keys
+        taken = owned(ScenarioCatalog).pluck(:key)
+        attributes[:key] = ScenarioCatalog.key_for(attributes[:name], taken) if attributes[:key].blank?
+        attributes[:name] = attributes[:key] if attributes[:name].blank?
+        catalog = ScenarioCatalog.new(attributes.merge(source_kind: "api"))
+        catalog.owner = current_owner
+        ScenarioCatalog.transaction do
+          catalog.save!
+          catalog.refresh_document!
+        end
+        catalog.sync_to_storage!
+
+        render json: { catalogs: [ ScenarioCatalogSerializer.full(catalog) ] }, status: :created
+      rescue ActiveRecord::RecordInvalid => e
+        render json: { errors: e.record.errors.full_messages }, status: :unprocessable_entity
       end
 
       # Imports from the connected repository params name. nil, with the
@@ -203,11 +254,33 @@ def import_from_repository
           return nil
         end
 
+        repository_import(connection, repository).call
+      end
+
+      # Previews what the connected repository params name would import. nil,
+      # with the refusal rendered, as #import_from_repository.
+      def preview_repository
+        connection = GithubConnection.for_owner(current_owner).first
+        if connection.nil?
+          render json: { error: "Connect GitHub in Settings first", code: "no_github_connection" }, status: :unprocessable_entity
+          return nil
+        end
+
+        repository = params[:repository].to_s
+        if connection.repository(repository).nil? && !ActionAgent.permitted?(current_user, :manage_github, connection)
+          permission_denied(:manage_github)
+          return nil
+        end
+
+        repository_import(connection, repository).preview
+      end
+
+      def repository_import(connection, repository)
         ScenarioCatalogRepositoryImport.new(
           owner: current_owner, connection: connection, repository: repository,
           ref: params[:ref].presence, path: params[:path].presence,
-          agents: owner_agents, projects: owned(Project)
-        ).call
+          agents: owner_agents, projects: owned(Project), mode: params[:mode].presence || "replace"
+        )
       end
 
       # Boots the project's sandbox when none is running, as the project's