Implementation Roadmap: N8N Killer Workflow Designer

📌 Quick Start for Developers

Clone This Approach

1. Read /Users/johnmcmahon/workspace/2025/valkyr/ValkyrAI/N8N_KILLER_WORKFLOW_INITIATIVE.md for full context
2. Read valor_inference_prompt.txt section "N8N KILLER WORKFLOW DESIGNER INITIATIVE" for golden rules
3. Follow the phase-by-phase breakdown below
4. Test continuously; never skip validation

---

Phase 1: UX/Usability Improvements (Days 1-2)

1.1 ExecModule Catalog & Schema Generation

Goal: Every ExecModule type (Email, REST, Stripe, etc.) has a JSON schema defining its configuration form.

File: execModuleCatalog.ts (MODIFY)

Add comprehensive module metadata including form schemas:

interface ModuleSchema {
  moduleType: string;
  className: string;
  displayName: string;
  description: string;
  category: "Communication" | "Data" | "External" | "Transform" | "Flow";
  color: string;
  icon: React.ReactNode;
  configFields: FormField[];
  examples: ExampleConfig[];
  requirements?: string[];
}

interface FormField {
  name: string;
  label: string;
  type:
    | "text"
    | "email"
    | "url"
    | "password"
    | "number"
    | "textarea"
    | "select"
    | "multiselect"
    | "json"
    | "apiLookup";
  required: boolean;
  description?: string;
  placeholder?: string;
  validation?: (value: any) => string | null;
  options?: Array<{ label: string; value: string }>;
  apiLookupConfig?: {
    apiEndpoint: string;
    labelField: string;
    valueField: string;
    searchFields: string[];
  };
  examples?: string[];
}

// Example: Email Module
const EMAIL_MODULE_SCHEMA: ModuleSchema = {
  moduleType: "Email",
  className: "com.valkyrlabs.workflow.modules.EmailModule",
  displayName: "Send Email",
  description: "Send emails to recipients with customizable subject and body",
  category: "Communication",
  color: "#a78bfa",
  icon: iconForExecModule("email"),
  configFields: [
    {
      name: "recipient",
      label: "Recipient Email",
      type: "email",
      required: true,
      description: "Email address to send to",
      placeholder: "user@example.com",
      validation: (v) =>
        /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(v) ? null : "Invalid email",
      apiLookupConfig: {
        apiEndpoint: "/User",
        labelField: "email",
        valueField: "email",
        searchFields: ["email", "name"],
      },
    },
    {
      name: "subject",
      label: "Subject",
      type: "text",
      required: true,
      description: "Email subject (supports {{variables}})",
      placeholder: "Order {{orderId}} confirmation",
      examples: ["Order {{orderId}} confirmation", "Welcome {{userName}}"],
    },
    {
      name: "body",
      label: "Body",
      type: "textarea",
      required: true,
      description: "Email body (supports {{variables}} and HTML)",
      placeholder: "Hi {{userName}}, your order has been confirmed.",
    },
    {
      name: "attachments",
      label: "Attachments",
      type: "multiselect",
      required: false,
      description: "File paths to attach",
      apiLookupConfig: {
        apiEndpoint: "/FileUpload",
        labelField: "filename",
        valueField: "filepath",
        searchFields: ["filename"],
      },
    },
  ],
  examples: [
    {
      name: "Order confirmation",
      config: {
        recipient: "{{user.email}}",
        subject: "Order {{orderId}} confirmed",
        body: "Thank you!",
      },
    },
  ],
};

// Define all 20+ modules similarly
export const ALL_MODULE_SCHEMAS: Map<string, ModuleSchema> = new Map([
  ["Email", EMAIL_MODULE_SCHEMA],
  ["REST", REST_MODULE_SCHEMA],
  ["Stripe", STRIPE_MODULE_SCHEMA],
  ["AWS", AWS_MODULE_SCHEMA],
  ["Slack", SLACK_MODULE_SCHEMA],
  // ... etc.
]);

// Helper functions
export function getModuleSchema(moduleType: string): ModuleSchema | undefined {
  return ALL_MODULE_SCHEMAS.get(moduleType);
}

export function getModuleConfigSchema(moduleType: string): any {
  const schema = getModuleSchema(moduleType);
  if (!schema) return null;

  // Convert to JSON Schema for validation
  return {
    type: "object",
    properties: Object.fromEntries(
      schema.configFields.map((f) => [
        f.name,
        {
          type: fieldTypeToJsonSchemaType(f.type),
          description: f.description,
          examples: f.examples,
        },
      ])
    ),
    required: schema.configFields.filter((f) => f.required).map((f) => f.name),
  };
}

Tests: execModuleCatalog.test.ts

describe("ExecModule Catalog", () => {
  it("should have schema for all registered module types", () => {
    const moduleTypes = ["Email", "REST", "Stripe", "AWS", "Slack"];
    moduleTypes.forEach((type) => {
      expect(getModuleSchema(type)).toBeDefined();
    });
  });

  it("should validate required fields", () => {
    const schema = getModuleSchema("Email");
    expect(
      schema?.configFields.some((f) => f.required && f.name === "recipient")
    ).toBe(true);
  });
});

1.2 Unified ExecModule Configuration Editor

Goal: Single form component that renders the right UI for any module type.

File: ExecModuleConfigBuilder.tsx (NEW)

import React, { useMemo } from "react";
import { ExecModule } from "@thorapi/model";
import { getModuleSchema, FormField } from "./execModuleCatalog";
import { Form, Alert } from "react-bootstrap";
import CoolButton from "@valkyr/component-library/CoolButton";

interface ExecModuleConfigBuilderProps {
  module: ExecModule;
  onChange: (updatedModule: ExecModule) => void;
  onValidationChange?: (errors: Record<string, string>) => void;
}

const ExecModuleConfigBuilder: React.FC<ExecModuleConfigBuilderProps> = ({
  module,
  onChange,
  onValidationChange,
}) => {
  const schema = useMemo(
    () => getModuleSchema(module.moduleType || module.className),
    [module.moduleType, module.className]
  );

  const [config, setConfig] = React.useState<Record<string, any>>(
    module.moduleData ? JSON.parse(module.moduleData) : {}
  );

  const [errors, setErrors] = React.useState<Record<string, string>>({});

  const handleFieldChange = (
    fieldName: string,
    value: any,
    field: FormField
  ) => {
    const newConfig = { ...config, [fieldName]: value };
    setConfig(newConfig);

    // Validate
    let error: string | null = null;
    if (field.required && (!value || value === "")) {
      error = `${field.label} is required`;
    } else if (field.validation && value) {
      error = field.validation(value);
    }

    const newErrors = { ...errors };
    if (error) {
      newErrors[fieldName] = error;
    } else {
      delete newErrors[fieldName];
    }
    setErrors(newErrors);
    onValidationChange?.(newErrors);

    // Update module
    const updated = {
      ...module,
      moduleData: JSON.stringify(newConfig),
    };
    onChange(updated);
  };

  if (!schema) {
    return <Alert variant="danger">Module schema not found</Alert>;
  }

  return (
    <div className="exec-module-config-builder">
      <h4>{schema.displayName}</h4>
      <p className="text-muted">{schema.description}</p>

      {Object.keys(errors).length > 0 && (
        <Alert variant="warning">
          <strong>Validation Errors:</strong>
          <ul>
            {Object.entries(errors).map(([field, error]) => (
              <li key={field}>{error}</li>
            ))}
          </ul>
        </Alert>
      )}

      <Form>
        {schema.configFields.map((field) => (
          <Form.Group key={field.name} className="mb-3">
            <Form.Label>
              {field.label}
              {field.required && <span className="text-danger">*</span>}
            </Form.Label>

            {field.type === "text" && (
              <Form.Control
                type="text"
                placeholder={field.placeholder}
                value={config[field.name] || ""}
                onChange={(e) =>
                  handleFieldChange(field.name, e.target.value, field)
                }
                isInvalid={!!errors[field.name]}
              />
            )}

            {field.type === "email" && (
              <Form.Control
                type="email"
                placeholder={field.placeholder}
                value={config[field.name] || ""}
                onChange={(e) =>
                  handleFieldChange(field.name, e.target.value, field)
                }
                isInvalid={!!errors[field.name]}
              />
            )}

            {field.type === "textarea" && (
              <Form.Control
                as="textarea"
                rows={4}
                placeholder={field.placeholder}
                value={config[field.name] || ""}
                onChange={(e) =>
                  handleFieldChange(field.name, e.target.value, field)
                }
                isInvalid={!!errors[field.name]}
              />
            )}

            {field.type === "select" && field.options && (
              <Form.Select
                value={config[field.name] || ""}
                onChange={(e) =>
                  handleFieldChange(field.name, e.target.value, field)
                }
                isInvalid={!!errors[field.name]}
              >
                <option value="">Select...</option>
                {field.options.map((opt) => (
                  <option key={opt.value} value={opt.value}>
                    {opt.label}
                  </option>
                ))}
              </Form.Select>
            )}

            {field.type === "json" && (
              <JsonEditor
                value={config[field.name] || {}}
                onChange={(value) =>
                  handleFieldChange(field.name, value, field)
                }
                isInvalid={!!errors[field.name]}
              />
            )}

            {field.type === "apiLookup" && field.apiLookupConfig && (
              <ApiLookupField
                config={field.apiLookupConfig}
                value={config[field.name] || ""}
                onChange={(value) =>
                  handleFieldChange(field.name, value, field)
                }
                isInvalid={!!errors[field.name]}
              />
            )}

            {field.description && (
              <Form.Text className="d-block mt-1 text-muted">
                {field.description}
              </Form.Text>
            )}

            {field.examples && (
              <Form.Text className="d-block mt-1">
                <strong>Examples:</strong> {field.examples.join(", ")}
              </Form.Text>
            )}

            {errors[field.name] && (
              <Form.Control.Feedback type="invalid" className="d-block">
                {errors[field.name]}
              </Form.Control.Feedback>
            )}
          </Form.Group>
        ))}
      </Form>
    </div>
  );
};

export default ExecModuleConfigBuilder;

1.3 Module Chaining Visualization

Goal: Show how ExecModules within a Task are chained, with data flow between them.

File: ModuleChainViewer.tsx (NEW)

import React from "react";
import { ExecModule, Task } from "@thorapi/model";
import { Handle, Position } from "reactflow";
import "./ModuleChainViewer.css";

interface ModuleChainViewerProps {
  task: Task;
  onModuleReorder?: (modules: ExecModule[]) => void;
  onModuleSelect?: (module: ExecModule) => void;
  selectedModuleId?: string;
}

const ModuleChainViewer: React.FC<ModuleChainViewerProps> = ({
  task,
  onModuleReorder,
  onModuleSelect,
  selectedModuleId,
}) => {
  const modules = React.useMemo(
    () =>
      (task.modules || []).sort(
        (a, b) => (a.moduleOrder || 0) - (b.moduleOrder || 0)
      ),
    [task.modules]
  );

  const [draggedModule, setDraggedModule] = React.useState<ExecModule | null>(
    null
  );
  const [dragOverIndex, setDragOverIndex] = React.useState<number | null>(null);

  const handleDragStart = (module: ExecModule) => {
    setDraggedModule(module);
  };

  const handleDragOver = (index: number) => {
    setDragOverIndex(index);
  };

  const handleDrop = (index: number) => {
    if (!draggedModule || !onModuleReorder) return;

    const fromIndex = modules.findIndex((m) => m.id === draggedModule.id);
    if (fromIndex === -1) return;

    const newModules = [...modules];
    newModules.splice(fromIndex, 1);
    newModules.splice(index, 0, draggedModule);

    // Re-order: 1.0, 2.0, 3.0, etc.
    newModules.forEach((m, i) => {
      m.moduleOrder = (i + 1) * 1.0;
    });

    onModuleReorder(newModules);
    setDraggedModule(null);
    setDragOverIndex(null);
  };

  return (
    <div className="module-chain-viewer">
      <h5>Module Chain (in execution order)</h5>
      <div className="chain-container">
        {modules.length === 0 ? (
          <div className="text-muted">No modules in this task</div>
        ) : (
          modules.map((module, index) => (
            <React.Fragment key={module.id}>
              <div
                className={`module-chain-item ${
                  selectedModuleId === module.id ? "selected" : ""
                } ${dragOverIndex === index ? "drag-over" : ""}`}
                draggable
                onDragStart={() => handleDragStart(module)}
                onDragOver={(e) => {
                  e.preventDefault();
                  handleDragOver(index);
                }}
                onDragLeave={() => setDragOverIndex(null)}
                onDrop={() => handleDrop(index)}
                onClick={() => onModuleSelect?.(module)}
              >
                <div className="module-chain-index">{index + 1}</div>
                <div className="module-chain-content">
                  <div className="module-chain-name">
                    {module.name || module.className?.split(".").pop()}
                  </div>
                  <div className="module-chain-order">
                    Order: {module.moduleOrder || 0}
                  </div>
                </div>
                <div className="module-chain-status">
                  <span
                    className={`status-badge status-${
                      module.status || "ready"
                    }`}
                  >
                    {module.status || "ready"}
                  </span>
                </div>
              </div>

              {index < modules.length - 1 && (
                <div className="module-chain-connector">
                  <div className="connector-arrow">↓</div>
                  <div className="connector-label">Output → Input</div>
                </div>
              )}
            </React.Fragment>
          ))
        )}
      </div>
    </div>
  );
};

export default ModuleChainViewer;

Styles: ModuleChainViewer.css (NEW)

.module-chain-viewer {
  padding: 12px;
  border: 1px solid #ddd;
  border-radius: 6px;
  background: #f9fafb;
}

.module-chain-viewer h5 {
  margin: 0 0 12px 0;
  font-size: 14px;
  font-weight: 600;
}

.chain-container {
  display: flex;
  flex-direction: column;
  gap: 8px;
}

.module-chain-item {
  display: flex;
  align-items: center;
  gap: 12px;
  padding: 10px 12px;
  border: 1px solid #ddd;
  border-radius: 4px;
  background: white;
  cursor: grab;
  transition: all 0.2s;
}

.module-chain-item:hover {
  border-color: #3b82f6;
  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
}

.module-chain-item.selected {
  border-color: #3b82f6;
  background: #eff6ff;
}

.module-chain-item.drag-over {
  border-color: #10b981;
  background: #ecfdf5;
}

.module-chain-index {
  display: flex;
  align-items: center;
  justify-content: center;
  width: 28px;
  height: 28px;
  border-radius: 50%;
  background: #3b82f6;
  color: white;
  font-weight: 600;
  font-size: 12px;
  flex-shrink: 0;
}

.module-chain-content {
  flex: 1;
}

.module-chain-name {
  font-weight: 500;
  font-size: 13px;
}

.module-chain-order {
  font-size: 12px;
  color: #999;
}

.module-chain-status {
  flex-shrink: 0;
}

.status-badge {
  display: inline-block;
  padding: 2px 8px;
  border-radius: 3px;
  font-size: 11px;
  font-weight: 600;
  text-transform: uppercase;
}

.status-ready {
  background: #dbeafe;
  color: #1e40af;
}

.status-running {
  background: #fef3c7;
  color: #92400e;
  animation: pulse 1.5s infinite;
}

.status-good {
  background: #dcfce7;
  color: #166534;
}

.status-error {
  background: #fee2e2;
  color: #991b1b;
}

.module-chain-connector {
  display: flex;
  flex-direction: column;
  align-items: center;
  gap: 4px;
  padding: 4px 0;
}

.connector-arrow {
  font-size: 14px;
  color: #9ca3af;
}

.connector-label {
  font-size: 10px;
  color: #9ca3af;
  text-transform: uppercase;
  font-weight: 600;
}

---

Continue with Remaining Tasks...

This is the foundation. Next steps include:

1. API Lookup Component - Auto-complete from REST APIs
2. API Browser Component - Browse and test available APIs
3. Workflow Canvas Updates - Integrate nested module visualization
4. Inspector Panel Improvements - Show module list with drag-reorder
5. Backend Service Tightening - Proper execution logic and state management
6. Workflow Trigger Endpoint - Allow external systems to trigger workflows
7. Full Test Suite - Unit and integration tests

---

Key Implementation Patterns

Pattern 1: New Object Creation (No ID!)

Frontend:

// Create workflow
const newWorkflow: Workflow = { name: "My Workflow" }; // NO id
const result = await createWorkflow(newWorkflow);
const workflowId = result.id; // Use server's ID

Backend:

@PostMapping("/Workflow")
public ResponseEntity<Workflow> createWorkflow(@RequestBody Workflow workflow) {
    // workflow.id should be null
    if (workflow.getId() != null) {
        throw new BadRequestException("ID must be null for creation");
    }
    Workflow saved = workflowService.saveOrUpdate(workflow);
    return ResponseEntity.created(...).body(saved);
}

Pattern 2: Async State Persistence

public CompletableFuture<Workflow> executeWorkflow(Workflow workflow) {
    // Within transaction, eagerly load
    Hibernate.initialize(workflow.getTasks());
    workflow.getTasks().forEach(t -> Hibernate.initialize(t.getModules()));

    // Get calling auth BEFORE async
    final Authentication callingAuth = SecurityContextHolder.getContext().getAuthentication();

    return CompletableFuture.supplyAsync(() -> {
        // Set auth in async thread
        SecurityContext ctx = SecurityContextHolder.createEmptyContext();
        ctx.setAuthentication(callingAuth);
        SecurityContextHolder.setContext(ctx);

        try {
            // Do work...
            persistWorkflowState(workflow);  // Uses REQUIRES_NEW
        } finally {
            SecurityContextHolder.clearContext();
        }
    });
}

@Transactional(propagation = Propagation.REQUIRES_NEW)
void persistWorkflowState(Workflow workflow) {
    workflowService.saveOrUpdate(workflow);
}

Pattern 3: Module Configuration Validation

const validateModuleConfig = (
  moduleType: string,
  config: Record<string, any>
) => {
  const schema = getModuleSchema(moduleType);
  if (!schema) return { valid: false, errors: ["Unknown module type"] };

  const errors: Record<string, string> = {};

  schema.configFields.forEach((field) => {
    if (field.required && (!config[field.name] || config[field.name] === "")) {
      errors[field.name] = `${field.label} is required`;
    }
    if (field.validation && config[field.name]) {
      const error = field.validation(config[field.name]);
      if (error) errors[field.name] = error;
    }
  });

  return {
    valid: Object.keys(errors).length === 0,
    errors,
  };
};

---

Testing Strategy

Unit Tests

• ExecModule catalog validation
• Form field validation
• Module chaining logic
• Workflow state transitions

Integration Tests

• Create workflow → add task → add modules → execute
• Module data flow (output of A → input of B)
• Branching and parallel execution
• Error handling

E2E Tests

• Create workflow in UI
• Configure all module types
• Save and reload
• Execute and monitor
• Check execution results

---

Success Metrics

✅ All 20+ module types have configuration forms
✅ Modules snap together visually
✅ Data flows correctly between modules
✅ Workflows save reliably
✅ No LazyInitializationException errors
✅ No JPA ID generation issues
✅ Tests pass (80%+ coverage)
✅ Users can create workflows in < 5 minutes

DEADLINE: Production-ready, zero bugs, enterprise-grade.