mirror of
https://github.com/dagu-org/dagu.git
synced 2025-12-28 06:34:22 +00:00
1314 lines
53 KiB
JSON
1314 lines
53 KiB
JSON
{
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"description": "Schema for Dagu DAG YAML format. Dagu uses YAML files to define Directed Acyclic Graphs (DAGs) for workflow orchestration.",
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"name": {
|
|
"type": "string",
|
|
"description": "Name of the DAG. If omitted, defaults to the YAML filename without extension."
|
|
},
|
|
"description": {
|
|
"type": "string",
|
|
"description": "A brief description explaining what this DAG does. This helps document the DAG's purpose."
|
|
},
|
|
"group": {
|
|
"type": "string",
|
|
"description": "An organizational label used to group related DAGs together. Useful for categorizing DAGs in the UI, e.g., 'DailyJobs', 'Analytics'."
|
|
},
|
|
"dotenv": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Path to a .env file to load environment variables from"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "List of paths to .env files to load environment variables from. Files can be specified as absolute paths, or relative to: DAG file directory, base config directory, or user's home directory."
|
|
}
|
|
],
|
|
"description": "Specifies .env files to load environment variables from. Defaults to [\".env\"] if not specified. Set to empty array [] to disable .env loading. Files are loaded relative to the DAG's workingDir."
|
|
},
|
|
"workingDir": {
|
|
"type": "string",
|
|
"description": "Working directory for the DAG. All relative paths (including dotenv files) are resolved relative to this directory. Defaults to the directory containing the DAG file."
|
|
},
|
|
"schedule": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Single cron expression for starting the DAG (e.g., '5 4 * * *' runs daily at 04:05)"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Multiple cron expressions for starting the DAG at different times"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"start": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Single cron expression for starting the DAG"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Multiple cron expressions for starting the DAG"
|
|
}
|
|
]
|
|
},
|
|
"stop": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Single cron expression for stopping the DAG"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Multiple cron expressions for stopping the DAG"
|
|
}
|
|
]
|
|
},
|
|
"restart": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Single cron expression for restarting the DAG"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Multiple cron expressions for restarting the DAG"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"additionalProperties": false,
|
|
"description": "Advanced scheduling with separate start, stop, and restart schedules"
|
|
}
|
|
],
|
|
"description": "Schedule configuration for the DAG. Can be a simple cron expression, multiple cron expressions, or an object with start/stop/restart schedules. If omitted, the DAG will only run manually."
|
|
},
|
|
"skipIfSuccessful": {
|
|
"type": "boolean",
|
|
"description": "When true, Dagu checks if this DAG has already succeeded since the last scheduled time. If it has, Dagu will skip the current scheduled run. This is useful for resource-intensive tasks or data processing jobs that shouldn't run twice. Note: Manual triggers always run regardless of this setting."
|
|
},
|
|
"tags": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Comma-separated list of tags for categorization"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "List of tags for categorization"
|
|
}
|
|
],
|
|
"description": "Tags for categorizing and searching DAGs. Useful for filtering and organizing DAGs."
|
|
},
|
|
"type": {
|
|
"type": "string",
|
|
"enum": ["graph", "chain", "agent"],
|
|
"default": "chain",
|
|
"description": "Execution type for steps. 'chain' (default) executes steps sequentially in the order they are defined, with each step automatically depending on the previous one. 'graph' uses dependency-based execution where steps run based on their 'depends' field. 'agent' is reserved for future agent-based execution."
|
|
},
|
|
"env": {
|
|
"oneOf": [
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"oneOf": [
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": true,
|
|
"description": "Map format: {KEY: value}"
|
|
},
|
|
{
|
|
"type": "string",
|
|
"pattern": "^[A-Za-z_][A-Za-z0-9_]*=.*$",
|
|
"description": "Key=Value format: 'KEY=value'"
|
|
}
|
|
]
|
|
},
|
|
"description": "Array of environment variable definitions (map format or Key=Value strings)"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": true,
|
|
"description": "Map format: {KEY1: value1, KEY2: value2}"
|
|
}
|
|
],
|
|
"description": "Environment variables available to all steps in the DAG. Supports three formats: 1) Map format: env: {KEY: value}, 2) Array of maps: env: [{KEY: value}], 3) Array of Key=Value strings: env: ['KEY=value'] (Docker/docker-compose compatible). Can use shell expansions (${VAR}), references to other environment variables, or command substitutions (`cmd`). Note: These won't be stored in execution history data for security."
|
|
},
|
|
"secrets": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/secretRef"
|
|
},
|
|
"description": "List of external secret references resolved at runtime and exposed as environment variables. Each entry declares the target environment variable name, provider, key, and optional provider-specific options."
|
|
},
|
|
"logDir": {
|
|
"type": "string",
|
|
"description": "Base directory for storing logs. Defaults to ${HOME}/.local/share/logs if not specified."
|
|
},
|
|
"logOutput": {
|
|
"type": "string",
|
|
"enum": ["separate", "merged"],
|
|
"default": "separate",
|
|
"description": "Controls how stdout and stderr are logged. 'separate' writes stdout to .out and stderr to .err files. 'merged' writes both to a single .log file with interleaved output."
|
|
},
|
|
"handlerOn": {
|
|
"type": "object",
|
|
"properties": {
|
|
"init": {
|
|
"$ref": "#/definitions/step",
|
|
"description": "Step to execute before any workflow steps run (after preconditions pass). If this fails, the DAG fails and no steps execute."
|
|
},
|
|
"failure": {
|
|
"$ref": "#/definitions/step"
|
|
},
|
|
"success": {
|
|
"$ref": "#/definitions/step"
|
|
},
|
|
"abort": {
|
|
"$ref": "#/definitions/step",
|
|
"description": "Step to execute when the DAG is aborted."
|
|
},
|
|
"exit": {
|
|
"$ref": "#/definitions/step"
|
|
}
|
|
},
|
|
"description": "Lifecycle event hooks that define commands to execute at various points in the DAG lifecycle: init (before steps), success, failure, abort, and exit (always runs last)."
|
|
},
|
|
"smtp": {
|
|
"type": "object",
|
|
"properties": {
|
|
"host": {
|
|
"type": "string",
|
|
"description": "SMTP server hostname"
|
|
},
|
|
"port": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "SMTP server port as string"
|
|
},
|
|
{
|
|
"type": "integer",
|
|
"description": "SMTP server port as number"
|
|
}
|
|
],
|
|
"description": "SMTP server port"
|
|
},
|
|
"username": {
|
|
"type": "string",
|
|
"description": "SMTP authentication username"
|
|
},
|
|
"password": {
|
|
"type": "string",
|
|
"description": "SMTP authentication password"
|
|
}
|
|
},
|
|
"description": "SMTP server configuration for sending email notifications."
|
|
},
|
|
"mailOn": {
|
|
"type": "object",
|
|
"properties": {
|
|
"failure": {
|
|
"type": "boolean",
|
|
"description": "Send email notification when DAG fails"
|
|
},
|
|
"success": {
|
|
"type": "boolean",
|
|
"description": "Send email notification when DAG succeeds"
|
|
}
|
|
},
|
|
"description": "Configuration for sending email notifications on DAG success or failure."
|
|
},
|
|
"errorMail": {
|
|
"$ref": "#/definitions/mailConfig",
|
|
"description": "Email configuration specifically for error notifications."
|
|
},
|
|
"infoMail": {
|
|
"$ref": "#/definitions/mailConfig",
|
|
"description": "Email configuration for informational notifications."
|
|
},
|
|
"timeoutSec": {
|
|
"type": "integer",
|
|
"description": "Maximum number of seconds allowed for the entire DAG to finish. If exceeded, the DAG is considered timed out."
|
|
},
|
|
"delaySec": {
|
|
"type": "integer",
|
|
"description": "Delay in seconds before starting the first node. Useful for staggering workloads."
|
|
},
|
|
"restartWaitSec": {
|
|
"type": "integer",
|
|
"description": "Number of seconds to wait before restarting a failed or stopped DAG. Typically used with a process supervisor."
|
|
},
|
|
"histRetentionDays": {
|
|
"type": "integer",
|
|
"description": "Number of days to retain execution history. After this period, older run logs/history can be purged."
|
|
},
|
|
"maxActiveRuns": {
|
|
"type": "integer",
|
|
"default": 1,
|
|
"description": "Maximum number of concurrent DAG runs allowed. Useful for limiting resource usage and preventing overload. If exceeded, new runs will be queued until existing ones complete. Defaults to 1. Set to -1 to disable queueing for this DAG."
|
|
},
|
|
"queue": {
|
|
"type": "string",
|
|
"description": "Name of the queue to assign this DAG to. If not specified, defaults to the DAG name. Used with global queue configuration to control concurrent execution across multiple DAGs."
|
|
},
|
|
"maxActiveSteps": {
|
|
"type": "integer",
|
|
"description": "Maximum number of concurrent steps that can be active at once. Useful for limiting resource usage."
|
|
},
|
|
"maxCleanUpTimeSec": {
|
|
"type": "integer",
|
|
"description": "Maximum time in seconds to spend cleaning up (stopping steps, finalizing logs) before forcing shutdown. If exceeded, processes will be killed."
|
|
},
|
|
"maxOutputSize": {
|
|
"type": "integer",
|
|
"default": 1048576,
|
|
"description": "Maximum size in bytes for the output of each step. If a step's output exceeds this limit, it will fail with an error. Defaults to 1MB (1048576 bytes). This limit also applies to pattern matching in preconditions and continueOn conditions."
|
|
},
|
|
"precondition": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/condition"
|
|
}
|
|
}
|
|
],
|
|
"description": "Conditions that must be satisfied before the DAG can run. Can use shell expansions or command substitutions to validate external states."
|
|
},
|
|
"preconditions": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/condition"
|
|
}
|
|
}
|
|
],
|
|
"description": "Alternative name for precondition. Works exactly the same way."
|
|
},
|
|
"params": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Space-separated positional parameters accessible as $1, $2, etc."
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": true
|
|
}
|
|
]
|
|
},
|
|
"description": "Named parameters as key-value pairs, accessible as ${KEY}"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": true,
|
|
"description": "Named parameters as key-value pairs, accessible as ${KEY}"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"schema": {
|
|
"type": "string",
|
|
"description": "Path to JSON Schema file (local file or remote URL) for parameter validation"
|
|
},
|
|
"values": {
|
|
"type": "object",
|
|
"additionalProperties": true,
|
|
"description": "Default parameter values to be validated against the schema"
|
|
}
|
|
},
|
|
"required": ["schema"],
|
|
"additionalProperties": false,
|
|
"description": "JSON Schema validation mode: validates parameters against a schema. 'schema' field is required, 'values' field is optional."
|
|
}
|
|
],
|
|
"description": "Default parameters that can be overridden when triggering the DAG. Supports three formats: 1) String format: space-separated positional parameters (e.g., 'arg1 arg2') accessible as $1, $2. 2) Array/Object format: named parameters as key-value pairs accessible as ${KEY}, supporting rich types (numbers, booleans, nested objects). 3) Schema validation format: object with 'schema' (path to JSON Schema file) and optional 'values' (parameter values) for runtime validation and defaults."
|
|
},
|
|
"container": {
|
|
"$ref": "#/definitions/container",
|
|
"description": "Default container configuration for all steps in the DAG. Steps can override this configuration with their own container settings."
|
|
},
|
|
"registryAuths": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Entire Docker authentication configuration as a JSON string, using the same format as DOCKER_AUTH_CONFIG or ~/.docker/config.json. Example: {\"auths\":{\"docker.io\":{\"auth\":\"base64(username:password)\"}}}"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Authentication value as a string. Can be: 1) base64-encoded 'username:password', 2) JSON string with username/password fields, or 3) environment variable reference like ${DOCKER_AUTH}"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"username": {
|
|
"type": "string",
|
|
"description": "Registry username. Supports environment variable expansion (e.g., ${DOCKER_USERNAME})."
|
|
},
|
|
"password": {
|
|
"type": "string",
|
|
"description": "Registry password or access token. Supports environment variable expansion (e.g., ${DOCKER_PASSWORD})."
|
|
},
|
|
"auth": {
|
|
"type": "string",
|
|
"description": "Pre-encoded authentication string. Should be base64(username:password) or a JSON string with credentials. Use this instead of username/password for pre-encoded credentials."
|
|
}
|
|
},
|
|
"description": "Authentication configuration object with username/password or pre-encoded auth string."
|
|
}
|
|
]
|
|
},
|
|
"description": "Map of registry hostnames to authentication configurations. Registry hostnames should match the registry in your image names (e.g., 'docker.io', 'ghcr.io', 'gcr.io', '123456789012.dkr.ecr.us-east-1.amazonaws.com')."
|
|
}
|
|
],
|
|
"description": "Authentication configuration for Docker container registries. Allows pulling images from private registries. Can be specified as a complete DOCKER_AUTH_CONFIG JSON string, or as a map of individual registry credentials. Authentication priority: 1) DAG-level registryAuths, 2) DOCKER_AUTH_CONFIG environment variable, 3) No authentication (public registries). Supports environment variable expansion with ${VAR} syntax."
|
|
},
|
|
"ssh": {
|
|
"$ref": "#/definitions/sshConfig",
|
|
"description": "Default SSH configuration for all steps in the DAG. Steps can override this configuration with their own SSH settings. Note: SSH and container configurations are mutually exclusive at the DAG level."
|
|
},
|
|
"steps": {
|
|
"oneOf": [
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/stepItem"
|
|
},
|
|
"description": "List of steps that define the DAG. Steps can be: 1) Step objects with full configuration, 2) Strings for simple commands (shorthand syntax), or 3) Nested arrays containing parallel steps that run simultaneously."
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": {
|
|
"$ref": "#/definitions/step"
|
|
},
|
|
"description": "Map of step names to step definitions. Steps can depend on each other, forming a directed acyclic graph."
|
|
}
|
|
]
|
|
},
|
|
"otel": {
|
|
"$ref": "#/definitions/otelConfig",
|
|
"description": "OpenTelemetry tracing configuration for distributed tracing of DAG execution."
|
|
},
|
|
"runConfig": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"disableParamEdit": {
|
|
"type": "boolean",
|
|
"default": false,
|
|
"description": "When true, prevents users from modifying DAG parameters when starting the DAG. Defaults to false (users can edit parameters)."
|
|
},
|
|
"disableRunIdEdit": {
|
|
"type": "boolean",
|
|
"default": false,
|
|
"description": "When true, prevents users from entering custom run IDs when starting the DAG. Defaults to false (users can enter custom run IDs)."
|
|
}
|
|
},
|
|
"description": "Configuration for controlling user interactions when starting DAG runs. Allows restricting parameter editing and custom run ID input."
|
|
},
|
|
"workerSelector": {
|
|
"type": "object",
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
},
|
|
"description": "Key-value pairs specifying worker label requirements for executing this DAG. The DAG will only run on workers that have all specified labels with matching values. For example: {\"gpu\": \"true\", \"memory\": \"64G\"} requires a worker with both gpu=true and memory=64G labels. This setting applies to the entire DAG; individual steps can override this with their own workerSelector."
|
|
},
|
|
"shell": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Shell command with optional arguments (e.g., '/bin/bash -e', 'bash -eo pipefail')"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Shell command as array (e.g., ['/bin/bash', '-e', '-o', 'pipefail'])"
|
|
}
|
|
],
|
|
"description": "Default shell to use for all steps in this DAG. Can be specified as a string with arguments (e.g., '/bin/bash -e') which will be automatically tokenized, or as an array for explicit argument separation. If not specified, the system default shell ($SHELL or /bin/sh) is used. Can be overridden at the step level."
|
|
}
|
|
},
|
|
"definitions": {
|
|
"secretRef": {
|
|
"type": "object",
|
|
"required": ["name", "provider", "key"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"name": {
|
|
"type": "string",
|
|
"description": "Environment variable name that will receive the secret value."
|
|
},
|
|
"provider": {
|
|
"type": "string",
|
|
"description": "Secret provider identifier (e.g., env, file, custom providers)."
|
|
},
|
|
"key": {
|
|
"type": "string",
|
|
"description": "Provider-specific key or identifier used to look up the secret."
|
|
},
|
|
"options": {
|
|
"type": "object",
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
},
|
|
"description": "Provider-specific configuration options. Values must be strings to support serialization."
|
|
}
|
|
},
|
|
"description": "Reference to an external secret that Dagu resolves at runtime. The resolved value is injected as an environment variable and masked in logs/output."
|
|
},
|
|
"stepItem": {
|
|
"oneOf": [
|
|
{
|
|
"$ref": "#/definitions/step"
|
|
},
|
|
{
|
|
"type": "string",
|
|
"description": "Shorthand syntax: a simple command string that will be converted to a step with 'command' field"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"oneOf": [
|
|
{
|
|
"$ref": "#/definitions/step"
|
|
},
|
|
{
|
|
"type": "string",
|
|
"description": "Shorthand syntax: a simple command string"
|
|
}
|
|
]
|
|
},
|
|
"description": "Nested array representing parallel steps. All steps in this array will run simultaneously, automatically depending on the previous sequential step, and the next sequential step will depend on all of them."
|
|
}
|
|
]
|
|
},
|
|
"step": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"name": {
|
|
"type": "string",
|
|
"description": "Unique identifier for the step within this DAG. If omitted, Dagu will automatically generate a name."
|
|
},
|
|
"dir": {
|
|
"type": "string",
|
|
"description": "Deprecated alias for workingDir. Still accepted for backward compatibility."
|
|
},
|
|
"id": {
|
|
"type": "string",
|
|
"description": "Optional short identifier for the step. Can be used in variable references like ${id.stdout} to access step properties. Must be unique within the DAG if specified."
|
|
},
|
|
"description": {
|
|
"type": "string",
|
|
"description": "Brief description of what this step does. Helps document the step's purpose."
|
|
},
|
|
"workingDir": {
|
|
"type": "string",
|
|
"description": "Working directory for the step. Inherits from DAG's workingDir if not specified. Overrides DAG-level workingDir for this step."
|
|
},
|
|
"executor": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"type": {
|
|
"type": "string",
|
|
"enum": [
|
|
"command",
|
|
"shell",
|
|
"docker",
|
|
"container",
|
|
"http",
|
|
"mail",
|
|
"ssh",
|
|
"jq",
|
|
"gha",
|
|
"github_action",
|
|
"github-action",
|
|
"dag",
|
|
"subworkflow",
|
|
"parallel",
|
|
"archive"
|
|
],
|
|
"description": "Type of executor to use for this step. 'command' is the default shell command executor. 'dag' is for sub-DAG execution. 'parallel' is for parallel step execution."
|
|
},
|
|
"config": {
|
|
"type": "object",
|
|
"additionalProperties": true,
|
|
"description": "Executor-specific configuration options."
|
|
}
|
|
},
|
|
"required": ["type"]
|
|
}
|
|
],
|
|
"description": "Specialized executor configuration for running the step (e.g., docker for containerized execution, http for API calls, mail for sending emails, gha for running GitHub Actions locally)."
|
|
},
|
|
"command": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Single command to execute"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": ["string", "number", "boolean"]
|
|
},
|
|
"description": "Multiple commands to execute sequentially. Each string is a separate command."
|
|
}
|
|
],
|
|
"description": "Command(s) to execute. Can be a single shell command string, or an array of commands to run sequentially (stops on first failure)."
|
|
},
|
|
"shell": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Shell command with optional arguments (e.g., '/bin/bash -e', 'bash -eo pipefail')"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Shell command as array (e.g., ['/bin/bash', '-e', '-o', 'pipefail'])"
|
|
}
|
|
],
|
|
"description": "Shell to use for executing the command. Can be specified as a string with arguments (e.g., '/bin/bash -e') which will be automatically tokenized, or as an array for explicit argument separation. Overrides DAG-level shell if specified. Defaults to DAG-level shell, or $SHELL/sh if not specified."
|
|
},
|
|
"shellPackages": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "List of packages to install before executing the step. Useful for ensuring dependencies are available. It's only available when the shell is nix-shell."
|
|
},
|
|
"script": {
|
|
"type": "string",
|
|
"description": "Multi-line script content that will be executed. Gets piped into the command if specified, otherwise uses default shell."
|
|
},
|
|
"stdout": {
|
|
"type": "string",
|
|
"description": "File path where the step's standard output (stdout) will be written."
|
|
},
|
|
"stderr": {
|
|
"type": "string",
|
|
"description": "File path where the step's standard error (stderr) will be written."
|
|
},
|
|
"logOutput": {
|
|
"type": "string",
|
|
"enum": ["separate", "merged"],
|
|
"description": "Override DAG-level log output mode. 'separate' writes stdout to .out and stderr to .err files. 'merged' writes both to a single .log file."
|
|
},
|
|
"output": {
|
|
"type": "string",
|
|
"description": "Variable name to capture the command's stdout. This output can be referenced in subsequent steps."
|
|
},
|
|
"depends": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Name of a step that must complete successfully before this step can start."
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "List of step names that must complete successfully before this step can start."
|
|
}
|
|
]
|
|
},
|
|
"continueOn": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"enum": ["skipped", "failed"],
|
|
"description": "Shorthand: 'skipped' to continue if step is skipped, 'failed' to continue if step fails"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"failure": {
|
|
"type": "boolean",
|
|
"description": "Continue dag-run even if this step fails"
|
|
},
|
|
"skipped": {
|
|
"type": "boolean",
|
|
"description": "Continue dag-run even if this step is skipped due to preconditions"
|
|
},
|
|
"exitCode": {
|
|
"oneOf": [
|
|
{
|
|
"type": "integer",
|
|
"description": "Exit code that should be treated as successful"
|
|
},
|
|
{
|
|
"type": "string",
|
|
"description": "Exit code represented as a string (parsed as integer at runtime)"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"description": "List of exit codes that should be treated as successful"
|
|
}
|
|
]
|
|
},
|
|
"output": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Output text or pattern that indicates success. Supports regex with 're:' prefix."
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"description": "Output text or patterns that indicate success. Supports regex with 're:' prefix."
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"markSuccess": {
|
|
"type": "boolean",
|
|
"description": "Mark the step as successful even if it technically failed but met continue conditions"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"description": "Conditions under which the DAG should continue executing even if this step fails or is skipped. Can be a string ('skipped' or 'failed') or an object with detailed configuration."
|
|
},
|
|
"retryPolicy": {
|
|
"type": "object",
|
|
"properties": {
|
|
"limit": {
|
|
"oneOf": [
|
|
{
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"type": "string"
|
|
}
|
|
],
|
|
"description": "Maximum number of retry attempts"
|
|
},
|
|
"intervalSec": {
|
|
"oneOf": [
|
|
{
|
|
"type": "integer"
|
|
},
|
|
{
|
|
"type": "string"
|
|
}
|
|
],
|
|
"description": "Seconds to wait between retry attempts"
|
|
},
|
|
"backoff": {
|
|
"oneOf": [
|
|
{
|
|
"type": "boolean",
|
|
"description": "When true, uses default multiplier of 2.0"
|
|
},
|
|
{
|
|
"type": "number",
|
|
"description": "Custom exponential backoff multiplier"
|
|
}
|
|
],
|
|
"description": "Exponential backoff multiplier for retry delays"
|
|
},
|
|
"maxIntervalSec": {
|
|
"type": "integer",
|
|
"description": "Maximum interval in seconds (caps exponential growth)"
|
|
},
|
|
"exitCode": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "integer"
|
|
},
|
|
"description": "List of exit codes that should trigger a retry. If not specified, all non-zero exit codes will trigger a retry."
|
|
}
|
|
},
|
|
"description": "Configuration for automatically retrying failed steps."
|
|
},
|
|
"repeatPolicy": {
|
|
"type": "object",
|
|
"properties": {
|
|
"repeat": {
|
|
"oneOf": [
|
|
{
|
|
"type": "boolean"
|
|
},
|
|
{
|
|
"type": "string",
|
|
"enum": ["while", "until"]
|
|
}
|
|
],
|
|
"description": "Determines if and how the step should be repeated. Can be a boolean or a string ('while' or 'until')."
|
|
},
|
|
"intervalSec": {
|
|
"type": "integer",
|
|
"description": "Interval in seconds between repetitions"
|
|
},
|
|
"limit": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "Maximum number of times this step will be executed. Once reached, the step stops repeating regardless of other conditions."
|
|
},
|
|
"backoff": {
|
|
"oneOf": [
|
|
{
|
|
"type": "boolean",
|
|
"description": "When true, uses default multiplier of 2.0"
|
|
},
|
|
{
|
|
"type": "number",
|
|
"description": "Custom exponential backoff multiplier"
|
|
}
|
|
],
|
|
"description": "Exponential backoff multiplier for repeat delays"
|
|
},
|
|
"maxIntervalSec": {
|
|
"type": "integer",
|
|
"description": "Maximum interval in seconds (caps exponential growth)"
|
|
},
|
|
"condition": {
|
|
"type": "string",
|
|
"description": "Command or expression to evaluate for repeat-until. Can include shell commands, environment variables, or command substitutions."
|
|
},
|
|
"expected": {
|
|
"type": "string",
|
|
"description": "Expected value or pattern to match against the condition result. Supports regex patterns with 're:' prefix."
|
|
},
|
|
"exitCode": {
|
|
"oneOf": [
|
|
{ "type": "integer" },
|
|
{ "type": "array", "items": { "type": "integer" } }
|
|
],
|
|
"description": "Exit code or list of exit codes that will stop repetition. If not specified, all non-zero exit codes will trigger repetition."
|
|
}
|
|
},
|
|
"description": "Configuration for repeatedly executing this step at fixed intervals or until a condition is met. Supports string matching, command substitution, and exit code checks."
|
|
},
|
|
"mailOnError": {
|
|
"type": "boolean",
|
|
"description": "Send an email notification if this specific step fails."
|
|
},
|
|
"precondition": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/condition"
|
|
}
|
|
}
|
|
],
|
|
"description": "Conditions that must be met before this step can run. Supports command exit codes, environment variables, and regex matching."
|
|
},
|
|
"preconditions": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/condition"
|
|
}
|
|
}
|
|
],
|
|
"description": "Alternative name for precondition. Works exactly the same way."
|
|
},
|
|
"signalOnStop": {
|
|
"type": "string",
|
|
"description": "Signal to send when stopping this step (e.g., SIGINT). If empty, uses same signal as parent process."
|
|
},
|
|
"timeoutSec": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Maximum execution time for the step in seconds. If set, this timeout takes precedence over the DAG-level timeout for this step."
|
|
},
|
|
"call": {
|
|
"type": "string",
|
|
"description": "Name of a DAG to execute as a sub dag-run."
|
|
},
|
|
"run": {
|
|
"type": "string",
|
|
"description": "Legacy field for DAG execution. Deprecated: use call instead."
|
|
},
|
|
"params": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Space-separated positional parameters accessible as $1, $2, etc."
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": true
|
|
}
|
|
]
|
|
},
|
|
"description": "Named parameters as key-value pairs"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": true,
|
|
"description": "Named parameters as key-value pairs"
|
|
}
|
|
],
|
|
"description": "Parameters for the step. Supports three formats: 1) String format: space-separated positional parameters (e.g., 'arg1 arg2') accessible as $1, $2. 2) Array format: named parameters as key-value objects preserving rich types (numbers, booleans, nested objects). 3) Object format: named parameters as simple key-value pairs. For sub-DAG execution (when 'call' or 'run' is specified), these are passed to the sub DAG and accessible as ${KEY}. For 'gha' executor, these are passed as action inputs (the 'with:' section in GitHub Actions syntax). Rich types are preserved when passing to executors that support them."
|
|
},
|
|
"parallel": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Variable containing array of items to process in parallel"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": true
|
|
}
|
|
]
|
|
},
|
|
"description": "Array of items to process in parallel"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"items": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Variable containing array of items"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "number"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": true
|
|
}
|
|
]
|
|
},
|
|
"description": "Array of items to process in parallel"
|
|
}
|
|
]
|
|
},
|
|
"maxConcurrent": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"maximum": 1000,
|
|
"default": 10,
|
|
"description": "Maximum number of concurrent executions (default: 10, maximum: 1000)"
|
|
}
|
|
},
|
|
"required": ["items"],
|
|
"additionalProperties": false,
|
|
"description": "Parallel execution configuration with concurrency control"
|
|
}
|
|
],
|
|
"description": "Configuration for parallel execution of sub DAGs. Only applicable when 'run' is specified. Allows processing multiple items concurrently using the same workflow definition."
|
|
},
|
|
"workerSelector": {
|
|
"type": "object",
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
},
|
|
"description": "Key-value pairs specifying worker label requirements for executing this step. The step will only run on workers that have all specified labels with matching values. For example: {\"gpu\": \"true\", \"memory\": \"64G\"} requires a worker with both gpu=true and memory=64G labels."
|
|
},
|
|
"env": {
|
|
"oneOf": [
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"oneOf": [
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": true,
|
|
"description": "Map format: {KEY: value}"
|
|
},
|
|
{
|
|
"type": "string",
|
|
"pattern": "^[A-Za-z_][A-Za-z0-9_]*=.*$",
|
|
"description": "Key=Value format: 'KEY=value'"
|
|
}
|
|
]
|
|
},
|
|
"description": "Array of environment variable definitions (map format or Key=Value strings)"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": true,
|
|
"description": "Map format: {KEY1: value1, KEY2: value2}"
|
|
}
|
|
],
|
|
"description": "Environment variables specific to this step. These override DAG-level environment variables with the same name. Supports three formats: 1) Map format: env: {KEY: value}, 2) Array of maps: env: [{KEY: value}], 3) Array of Key=Value strings: env: ['KEY=value'] (Docker/docker-compose compatible). Can use shell expansions (${VAR}), references to other environment variables, or command substitutions (`cmd`). When used with 'container' field, these are merged with container.env."
|
|
},
|
|
"container": {
|
|
"$ref": "#/definitions/container",
|
|
"description": "Container configuration for this step. When specified, the step runs in its own container instead of the DAG-level container. Uses the same configuration format as the DAG-level container field. Note: Cannot be used together with 'executor'."
|
|
}
|
|
}
|
|
},
|
|
"condition": {
|
|
"type": "object",
|
|
"properties": {
|
|
"condition": {
|
|
"type": "string",
|
|
"description": "Command or expression to evaluate. Can include shell commands, environment variables, or command substitutions with backticks."
|
|
},
|
|
"command": {
|
|
"type": "string",
|
|
"description": "Deprecated alias for condition. Still accepted for backward compatibility."
|
|
},
|
|
"expected": {
|
|
"type": "string",
|
|
"description": "Expected value or pattern to match against the condition result. Supports regex patterns with 're:' prefix (e.g., 're:0[1-9]' for matching numbers 01-09)."
|
|
},
|
|
"negate": {
|
|
"type": "boolean",
|
|
"description": "If true, inverts the condition result. The DAG/step will run when the condition does NOT match the expected value."
|
|
}
|
|
},
|
|
"description": "Defines a condition that must be met before execution. Used in preconditions at both DAG and step levels."
|
|
},
|
|
"mailConfig": {
|
|
"type": "object",
|
|
"properties": {
|
|
"from": {
|
|
"type": "string",
|
|
"description": "Email address to use as the sender address for notifications."
|
|
},
|
|
"to": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string",
|
|
"description": "Single email address to receive notifications."
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "List of email addresses to receive notifications."
|
|
}
|
|
],
|
|
"description": "Email address(es) to receive notifications. Can be a single string or an array of strings."
|
|
},
|
|
"prefix": {
|
|
"type": "string",
|
|
"description": "Text to prepend to the email subject line. Useful for filtering or categorizing notification emails."
|
|
},
|
|
"attachLogs": {
|
|
"type": "boolean",
|
|
"description": "When true, relevant log files will be attached to the notification email."
|
|
}
|
|
},
|
|
"description": "Configuration for email notifications, used by errorMail and infoMail settings."
|
|
},
|
|
"otelConfig": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"enabled": {
|
|
"type": "boolean",
|
|
"description": "Enable or disable OpenTelemetry tracing for this DAG. When false, no traces will be exported."
|
|
},
|
|
"endpoint": {
|
|
"type": "string",
|
|
"description": "OTLP endpoint URL for exporting traces. Supports both gRPC (e.g., http://localhost:4317) and HTTP (e.g., http://localhost:4318/v1/traces) protocols."
|
|
},
|
|
"headers": {
|
|
"type": "object",
|
|
"additionalProperties": true,
|
|
"description": "Custom headers to include with OTLP export requests. Useful for authentication (e.g., Authorization: Bearer token)."
|
|
},
|
|
"insecure": {
|
|
"type": "boolean",
|
|
"default": false,
|
|
"description": "Skip TLS certificate verification when connecting to the OTLP endpoint. Only use in development environments."
|
|
},
|
|
"timeout": {
|
|
"type": "string",
|
|
"description": "Timeout for exporting traces to the OTLP endpoint. Accepts duration strings like '30s', '1m', '5m'."
|
|
},
|
|
"resource": {
|
|
"type": "object",
|
|
"additionalProperties": true,
|
|
"properties": {
|
|
"service.name": {
|
|
"type": "string",
|
|
"description": "Service name for the traces. Defaults to 'dagu' if not specified. Can use variables like ${DAG_NAME}."
|
|
},
|
|
"service.version": {
|
|
"type": "string",
|
|
"description": "Version of the service producing the traces."
|
|
},
|
|
"deployment.environment": {
|
|
"type": "string",
|
|
"description": "Deployment environment (e.g., production, staging, development)."
|
|
}
|
|
},
|
|
"description": "Resource attributes to attach to all spans. Common attributes include service.name, service.version, and deployment.environment."
|
|
}
|
|
},
|
|
"description": "OpenTelemetry configuration for distributed tracing. Enables detailed execution traces for DAGs and steps, providing visibility into workflow performance and debugging."
|
|
},
|
|
"container": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"allOf": [
|
|
{
|
|
"if": {
|
|
"properties": { "startup": { "const": "command" } },
|
|
"required": ["startup"]
|
|
},
|
|
"then": {
|
|
"required": ["command"],
|
|
"properties": {
|
|
"command": { "minItems": 1 }
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"properties": {
|
|
"name": {
|
|
"type": "string",
|
|
"description": "Custom container name. If empty, Docker generates a random name. Must be unique - if a container with this name already exists (running or stopped), the DAG will fail."
|
|
},
|
|
"image": {
|
|
"type": "string",
|
|
"description": "Container image to use (e.g., 'python:3.11', 'node:20'). Required when using container configuration."
|
|
},
|
|
"pullPolicy": {
|
|
"oneOf": [
|
|
{
|
|
"type": "boolean",
|
|
"description": "Boolean shorthand: true = 'always', false = 'never'"
|
|
},
|
|
{
|
|
"type": "string",
|
|
"enum": ["always", "never", "missing"],
|
|
"description": "String policy: 'always' pulls the image every time, 'never' uses only local images, 'missing' pulls only if not present locally"
|
|
}
|
|
],
|
|
"default": "missing",
|
|
"description": "Image pull policy. Accepts boolean (true='always', false='never') or string ('always', 'never', 'missing'). Default is 'missing'."
|
|
},
|
|
"env": {
|
|
"oneOf": [
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"oneOf": [
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": true
|
|
},
|
|
{
|
|
"type": "string",
|
|
"pattern": "^[A-Za-z_][A-Za-z0-9_]*=.*$"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": true
|
|
}
|
|
],
|
|
"description": "Environment variables to set in the container. Supports maps or arrays combining maps and KEY=value strings. Values can include variable expansion (e.g., 'API_KEY=${API_KEY}')."
|
|
},
|
|
"volumes": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Volume mounts for the container. Format: 'host_path:container_path[:mode]' where mode is 'ro' for read-only or 'rw' (default) for read-write. Relative paths (starting with './' or '.') are resolved relative to the DAG's workingDir."
|
|
},
|
|
"user": {
|
|
"type": "string",
|
|
"description": "User (and optionally group) to run the container as. Format: 'user' or 'user:group' (e.g., '1000' or '1000:1000')."
|
|
},
|
|
"workingDir": {
|
|
"type": "string",
|
|
"description": "Working directory inside the container. Commands will be executed from this directory."
|
|
},
|
|
"workDir": {
|
|
"type": "string",
|
|
"description": "Deprecated alias for workingDir. Still accepted for backward compatibility."
|
|
},
|
|
"platform": {
|
|
"type": "string",
|
|
"description": "Platform for the container (e.g., 'linux/amd64', 'linux/arm64'). Useful for ensuring compatibility on different architectures."
|
|
},
|
|
"ports": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Ports to expose from the container. Format: 'host_port:container_port' or just 'container_port'."
|
|
},
|
|
"network": {
|
|
"type": "string",
|
|
"description": "Network mode for the container (e.g., 'host', 'bridge', 'none', or a custom network name)."
|
|
},
|
|
"startup": {
|
|
"type": "string",
|
|
"enum": ["keepalive", "entrypoint", "command"],
|
|
"default": "keepalive",
|
|
"description": "Startup mode for the DAG-level container: 'keepalive' uses an internal keepalive/sleep, 'entrypoint' honors the image's ENTRYPOINT/CMD, 'command' runs the provided command array."
|
|
},
|
|
"command": {
|
|
"type": "array",
|
|
"items": { "type": "string" },
|
|
"description": "Command to execute when startup is 'command'. Must be a non-empty array."
|
|
},
|
|
"waitFor": {
|
|
"type": "string",
|
|
"enum": ["running", "healthy"],
|
|
"default": "running",
|
|
"description": "Readiness condition before steps execute: 'running' waits for container to run; 'healthy' waits for Docker healthcheck to report healthy."
|
|
},
|
|
"logPattern": {
|
|
"type": "string",
|
|
"description": "Optional regex; if set, waits for this pattern to appear in container logs before steps run."
|
|
},
|
|
"restartPolicy": {
|
|
"type": "string",
|
|
"description": "Docker restart policy for the container: 'no', 'always', or 'unless-stopped'."
|
|
},
|
|
"keepContainer": {
|
|
"type": "boolean",
|
|
"default": false,
|
|
"description": "Keep the container running after the step completes. Useful for debugging or when running services that need to persist between steps."
|
|
}
|
|
},
|
|
"required": ["image"],
|
|
"description": "Container configuration that can be specified at DAG or step level. When used, automatically sets the executor to docker."
|
|
},
|
|
"sshConfig": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"user": {
|
|
"type": "string",
|
|
"description": "SSH username for authentication."
|
|
},
|
|
"host": {
|
|
"type": "string",
|
|
"description": "Hostname or IP address of the SSH server."
|
|
},
|
|
"port": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "integer"
|
|
}
|
|
],
|
|
"default": "22",
|
|
"description": "SSH port number. Defaults to 22."
|
|
},
|
|
"key": {
|
|
"type": "string",
|
|
"description": "Path to the SSH private key file. If not specified, defaults are tried: ~/.ssh/id_rsa, ~/.ssh/id_ecdsa, ~/.ssh/id_ed25519, ~/.ssh/id_dsa."
|
|
},
|
|
"password": {
|
|
"type": "string",
|
|
"description": "SSH password for authentication (not recommended; prefer key-based auth)."
|
|
},
|
|
"strictHostKey": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Enable strict host key checking. When true (default), verifies the server's host key against known_hosts. Set to false only for testing."
|
|
},
|
|
"knownHostFile": {
|
|
"type": "string",
|
|
"default": "~/.ssh/known_hosts",
|
|
"description": "Path to the known_hosts file for host key verification. Defaults to ~/.ssh/known_hosts."
|
|
}
|
|
},
|
|
"required": ["user", "host"],
|
|
"description": "SSH configuration for remote command execution. Can be specified at DAG level for reuse across steps, or at step level for specific overrides. Password authentication is supported but key-based auth is recommended."
|
|
}
|
|
}
|
|
}
|