models:
  - name: raw_s3_orpheus_ddb
    description: |
      External view of Orpheus data from the S3 dump of Orpheus DynamoDB data source.
      This model directly reads from S3 JSON files containing DynamoDB stream events
      for Orpheus interactions. Data is parsed from DynamoDB's NewImage format and
      includes all raw fields as they appear in the source system.
    config:
      group: create
      tags: ['partition_type=hourly', 'automation_condition=hourly_cron_with_eager_historical_backfill_condition']
    meta:
      dagster:
        upstream_glue_job: 'ddb_orpheus' 
        partition_start_date: "2025-10-06"  # Matches ORPHEUS_PROJECT_START_DATE
        partition_start_hour: 16
        end_offset: 0
        backfill_policy:
          max_partitions_per_run: -1
    columns:
      - name: id
        description: Unique identifier for the Orpheus record (from DynamoDB Keys:item_id)
      - name: type
        description: Event type with timestamp suffix (from DynamoDB Keys:type)
      - name: message_id
        description: Unique identifier for the message (from DynamoDB NewImage:message_id)
      - name: session_id
        description: Session identifier for the conversation (from DynamoDB NewImage:session_id)
      - name: name
        description: Name or label associated with the chat (from DynamoDB NewImage:name)
      - name: content
        description: Raw content of the message or event (from DynamoDB NewImage:content)
      - name: finish_reason
        description: Reason why the AI response was finished (from DynamoDB NewImage:finish_reason)
      - name: sessions
        description: Session-related data as variant JSON (from DynamoDB NewImage:sessions)
      - name: tool_call_id
        description: Tool call identifier (from DynamoDB NewImage:tool_call_id)
      - name: tool_calls
        description: Tool calls data as variant JSON array (from DynamoDB NewImage:tool_calls)
      - name: user_id
        description: User ID who initiated the conversation (from DynamoDB NewImage:user_id)
      - name: visibility
        description: Visibility setting for the event or message (from DynamoDB NewImage:visibility)
      - name: workspace_id
        description: Workspace identifier where the conversation took place (from DynamoDB NewImage:workspace_id)
      - name: created_at
        description: Creation timestamp as float (from DynamoDB NewImage:created_at)
      - name: updated_at
        description: Last update timestamp as float (from DynamoDB NewImage:updated_at)
      - name: p_date
        description: Date partition parsed from S3 file path (pdate=YYYY-MM-DD)
      - name: p_hour
        description: Hour partition parsed from S3 file path (phour=0-23)

  - name: raw_orpheus_ddb
    description: |
      Incrementally loaded data from the Orpheus DynamoDB S3 dump.
      This model provides a clean, incremental view of the raw S3 data with proper
      partitioning and clustering for efficient querying. It serves as the foundation
      for all downstream Orpheus staging models.
    config:
      group: create
      tags: ['partition_type=hourly', 'automation_condition=hourly_cron_with_eager_historical_backfill_condition']
      on_schema_change: append_new_columns
    meta:
      dagster:
        partition_start_date: "2025-10-06"  # Matches ORPHEUS_PROJECT_START_DATE
        partition_start_hour: 16
        end_offset: 0
        backfill_policy:
          max_partitions_per_run: 24
    columns:
      - name: id
        description: Unique identifier for the Orpheus record
        tests:
          - not_null
      - name: type
        description: Event type with timestamp suffix and message_id (e.g., 'message#1234567890.123456#sadjhfkajsdf')
      - name: message_id
        description: Unique identifier for the message
      - name: session_id
        description: Session identifier for the conversation
      - name: name
        description: Name or label associated with the chat
      - name: content
        description: Raw content of the message or event
      - name: finish_reason
        description: Reason why the AI response was finished (null for user messages)
      - name: sessions
        description: Session-related data as variant JSON
      - name: tool_call_id
        description: Tool call identifier used in this event
      - name: tool_calls
        description: Tool calls data as variant JSON array containing function calls and arguments
      - name: user_id
        description: User ID who initiated the conversation (Clerk user Id, not normalized user_id)
      - name: visibility
        description: Visibility setting for the event or message
      - name: workspace_id
        description: Workspace identifier where the conversation took place
      - name: created_at
        description: Creation timestamp as float (microseconds since epoch)
      - name: updated_at
        description: Last update timestamp as float (microseconds since epoch)
      - name: p_date
        description: Date partition for data organization and query optimization
        tests:
          - not_null
      - name: p_hour
        description: Hour partition for data organization and query optimization
        tests:
          - not_null