PLAICraft Multimodal Gameplay Dataset
type: Dataset
description: Time-aligned video, audio, action data collected from multi-player Minecraft World, and pre-encoded latents for every gameplay session, generated from actual files in an S3 bucket. This is an initial release of a small subset out of the 10,000 hrs of data that are security check passed, more will be released in the future.
conforms to: http://mlcommons.org/croissant/1.0
cite as: PLAICraft
date published: 2025-05-10T00:00:00
keywords: Minecraft
keywords: multimodal
keywords: game AI
keywords: Emboddied AI
keywords: video
keywords: audio
keywords: action
license: https://creativecommons.org/licenses/by/4.0/
version: 1.0.0
distribution: sqlite
distribution: audio_in
distribution: audio_out
distribution: event_log
distribution: positions
distribution: videos
distribution: enc_audio
distribution: enc_video
record set: sqlite
record set: audio_in
record set: audio_out
record set: event_log
record set: positions
record set: videos
record set: enc_audio
record set: enc_video

sqlite
type: FileSet
description: SQLite session DB containing the mouse and keyboard action data from the Forge mod, with session metadata
encoding format: application/octet-stream

audio_in
type: FileSet
description: Player microphone input (WAV)
encoding format: application/octet-stream

audio_out
type: FileSet
description: Game-audio output (WAV)
encoding format: application/octet-stream

event_log
type: FileSet
description: Event log (JSON) containing the system level key loggings
encoding format: application/octet-stream

positions
type: FileSet
description: Position stream (JSON) (not being used)
encoding format: application/octet-stream

videos
type: FileSet
description: Screen-capture video (MP4)
encoding format: application/octet-stream

enc_audio
type: FileSet
description: Encoded audio latents (HDF5)
encoding format: application/octet-stream

enc_video
type: FileSet
description: Encoded video latents (PT)
encoding format: application/octet-stream

sqlite
type: RecordSet
description: All sqlite files in the dataset
field: path
field: player
field: session

path
type: Field
data type: Text
source:
  file set: sqlite
  extract:
    file property: filename

player
type: Field
data type: Text
source:
  file set: sqlite
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

session
type: Field
data type: Text
source:
  file set: sqlite
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

audio_in
type: RecordSet
description: All audio_in files in the dataset
field: path
field: player
field: session

path
type: Field
data type: Text
source:
  file set: audio_in
  extract:
    file property: filename

player
type: Field
data type: Text
source:
  file set: audio_in
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

session
type: Field
data type: Text
source:
  file set: audio_in
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

audio_out
type: RecordSet
description: All audio_out files in the dataset
field: path
field: player
field: session

path
type: Field
data type: Text
source:
  file set: audio_out
  extract:
    file property: filename

player
type: Field
data type: Text
source:
  file set: audio_out
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

session
type: Field
data type: Text
source:
  file set: audio_out
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

event_log
type: RecordSet
description: All event_log files in the dataset
field: path
field: player
field: session

path
type: Field
data type: Text
source:
  file set: event_log
  extract:
    file property: filename

player
type: Field
data type: Text
source:
  file set: event_log
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

session
type: Field
data type: Text
source:
  file set: event_log
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

positions
type: RecordSet
description: All positions files in the dataset
field: path
field: player
field: session

path
type: Field
data type: Text
source:
  file set: positions
  extract:
    file property: filename

player
type: Field
data type: Text
source:
  file set: positions
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

session
type: Field
data type: Text
source:
  file set: positions
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

videos
type: RecordSet
description: All videos files in the dataset
field: path
field: player
field: session

path
type: Field
data type: Text
source:
  file set: videos
  extract:
    file property: filename

player
type: Field
data type: Text
source:
  file set: videos
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

session
type: Field
data type: Text
source:
  file set: videos
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

enc_audio
type: RecordSet
description: All enc_audio files in the dataset
field: path
field: player
field: session

path
type: Field
data type: Text
source:
  file set: enc_audio
  extract:
    file property: filename

player
type: Field
data type: Text
source:
  file set: enc_audio
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

session
type: Field
data type: Text
source:
  file set: enc_audio
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

enc_video
type: RecordSet
description: All enc_video files in the dataset
field: path
field: player
field: session

path
type: Field
data type: Text
source:
  file set: enc_video
  extract:
    file property: filename

player
type: Field
data type: Text
source:
  file set: enc_video
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*

session
type: Field
data type: Text
source:
  file set: enc_video
  extract:
    file property: filename
  transform:
    regex: .*/([0-9a-f]{64})/([0-9]{4,})/.*